diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index a823829419a05d..4e7a8d761891c9 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,9 +1,8 @@ FROM mcr.microsoft.com/vscode/devcontainers/rust:1-bullseye -# Install cmake and protobuf-compiler +# Install cmake RUN apt-get update \ && apt-get install -y cmake \ - && apt-get install -y protobuf-compiler \ && rm -rf /var/lib/apt/lists/* # Install Deno diff --git a/.dprint.json b/.dprint.json index 6ee3db3c5ddf2e..b9c2d1ebc14313 100644 --- a/.dprint.json +++ b/.dprint.json @@ -31,6 +31,8 @@ "cli/tsc/dts/lib.scripthost.d.ts", "cli/tsc/dts/lib.webworker*.d.ts", "cli/tsc/dts/typescript.d.ts", + "cli/tools/doc/prism.css", + "cli/tools/doc/prism.js", "ext/websocket/autobahn/reports", "gh-pages", "target", @@ -65,13 +67,17 @@ "tests/wpt/runner/expectation.json", "tests/wpt/runner/manifest.json", "tests/wpt/suite", - "third_party" + "third_party", + "tests/specs/run/shebang_with_json_imports_tsc", + "tests/specs/run/shebang_with_json_imports_swc", + "tests/specs/run/ext_flag_takes_precedence_over_extension", + "tests/specs/run/error_syntax_empty_trailing_line/error_syntax_empty_trailing_line.mjs" ], "plugins": [ - "https://plugins.dprint.dev/typescript-0.93.0.wasm", - "https://plugins.dprint.dev/json-0.19.3.wasm", + "https://plugins.dprint.dev/typescript-0.93.2.wasm", + "https://plugins.dprint.dev/json-0.19.4.wasm", "https://plugins.dprint.dev/markdown-0.17.8.wasm", - "https://plugins.dprint.dev/toml-0.6.2.wasm", + "https://plugins.dprint.dev/toml-0.6.3.wasm", "https://plugins.dprint.dev/exec-0.5.0.json@8d9972eee71fa1590e04873540421f3eda7674d0f1aae3d7c788615e7b7413d0", "https://plugins.dprint.dev/g-plane/pretty_yaml-v0.5.0.wasm" ] diff --git a/.github/workflows/cargo_publish.yml b/.github/workflows/cargo_publish.yml index 45f075b5796995..3af97f4662d1ea 100644 --- a/.github/workflows/cargo_publish.yml +++ b/.github/workflows/cargo_publish.yml @@ -2,10 +2,15 @@ name: cargo_publish on: workflow_dispatch +# Ensures only one publish is running at a time +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + jobs: build: name: cargo publish - runs-on: ubuntu-20.04-xl + runs-on: ubuntu-24.04-xl timeout-minutes: 90 env: @@ -28,16 +33,10 @@ jobs: - uses: dsherret/rust-toolchain-file@v1 - name: Install deno - uses: denoland/setup-deno@v1 + uses: denoland/setup-deno@v2 with: deno-version: v1.x - - name: Install protoc - uses: arduino/setup-protoc@v3 - with: - version: '21.12' - repo-token: '${{ secrets.GITHUB_TOKEN }}' - - name: Publish env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index 35020a5f8c55b8..9a166e6cf02d99 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -5,15 +5,16 @@ import { stringify } from "jsr:@std/yaml@^0.221/stringify"; // Bump this number when you want to purge the cache. // Note: the tools/release/01_bump_crate_versions.ts script will update this version // automatically via regex, so ensure that this line maintains this format. -const cacheVersion = 15; +const cacheVersion = 28; -const ubuntuX86Runner = "ubuntu-22.04"; -const ubuntuX86XlRunner = "ubuntu-22.04-xl"; +const ubuntuX86Runner = "ubuntu-24.04"; +const ubuntuX86XlRunner = "ubuntu-24.04-xl"; const ubuntuARMRunner = "ubicloud-standard-16-arm"; const windowsX86Runner = "windows-2022"; const windowsX86XlRunner = "windows-2022-xl"; const macosX86Runner = "macos-13"; const macosArmRunner = "macos-14"; +const selfHostedMacosArmRunner = "self-hosted"; const Runners = { linuxX86: { @@ -40,7 +41,8 @@ const Runners = { macosArm: { os: "macos", arch: "aarch64", - runner: macosArmRunner, + runner: + `\${{ github.repository == 'denoland/deno' && startsWith(github.ref, 'refs/tags/') && '${selfHostedMacosArmRunner}' || '${macosArmRunner}' }}`, }, windowsX86: { os: "windows", @@ -59,7 +61,7 @@ const prCacheKeyPrefix = `${cacheVersion}-cargo-target-\${{ matrix.os }}-\${{ matrix.arch }}-\${{ matrix.profile }}-\${{ matrix.job }}-`; // Note that you may need to add more version to the `apt-get remove` line below if you change this -const llvmVersion = 18; +const llvmVersion = 19; const installPkgsCommand = `sudo apt-get install --no-install-recommends clang-${llvmVersion} lld-${llvmVersion} clang-tools-${llvmVersion} clang-format-${llvmVersion} clang-tidy-${llvmVersion}`; const sysRootStep = { @@ -71,7 +73,7 @@ export DEBIAN_FRONTEND=noninteractive sudo apt-get -qq remove --purge -y man-db > /dev/null 2> /dev/null # Remove older clang before we install sudo apt-get -qq remove \ - 'clang-12*' 'clang-13*' 'clang-14*' 'clang-15*' 'clang-16*' 'llvm-12*' 'llvm-13*' 'llvm-14*' 'llvm-15*' 'llvm-16*' 'lld-12*' 'lld-13*' 'lld-14*' 'lld-15*' 'lld-16*' > /dev/null 2> /dev/null + 'clang-12*' 'clang-13*' 'clang-14*' 'clang-15*' 'clang-16*' 'clang-17*' 'clang-18*' 'llvm-12*' 'llvm-13*' 'llvm-14*' 'llvm-15*' 'llvm-16*' 'lld-12*' 'lld-13*' 'lld-14*' 'lld-15*' 'lld-16*' 'lld-17*' 'lld-18*' > /dev/null 2> /dev/null # Install clang-XXX, lld-XXX, and debootstrap. echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${llvmVersion} main" | @@ -86,7 +88,7 @@ ${installPkgsCommand} || echo 'Failed. Trying again.' && sudo apt-get clean && s (yes '' | sudo update-alternatives --force --all) > /dev/null 2> /dev/null || true echo "Decompressing sysroot..." -wget -q https://github.com/denoland/deno_sysroot_build/releases/download/sysroot-20240528/sysroot-\`uname -m\`.tar.xz -O /tmp/sysroot.tar.xz +wget -q https://github.com/denoland/deno_sysroot_build/releases/download/sysroot-20241030/sysroot-\`uname -m\`.tar.xz -O /tmp/sysroot.tar.xz cd / xzcat /tmp/sysroot.tar.xz | sudo tar -x sudo mount --rbind /dev /sysroot/dev @@ -191,14 +193,9 @@ const installNodeStep = { uses: "actions/setup-node@v4", with: { "node-version": 18 }, }; -const installProtocStep = { - name: "Install protoc", - uses: "arduino/setup-protoc@v3", - with: { "version": "21.12", "repo-token": "${{ secrets.GITHUB_TOKEN }}" }, -}; const installDenoStep = { name: "Install Deno", - uses: "denoland/setup-deno@v1", + uses: "denoland/setup-deno@v2", with: { "deno-version": "v1.x" }, }; @@ -494,7 +491,6 @@ const ci = { if: "matrix.job == 'bench' || matrix.job == 'test'", ...installNodeStep, }, - installProtocStep, { if: [ "matrix.profile == 'release' &&", @@ -758,8 +754,10 @@ const ci = { run: [ "cd target/release", "zip -r deno-${{ matrix.arch }}-unknown-linux-gnu.zip deno", + "shasum -a 256 deno-${{ matrix.arch }}-unknown-linux-gnu.zip > deno-${{ matrix.arch }}-unknown-linux-gnu.zip.sha256sum", "strip denort", "zip -r denort-${{ matrix.arch }}-unknown-linux-gnu.zip denort", + "shasum -a 256 denort-${{ matrix.arch }}-unknown-linux-gnu.zip > denort-${{ matrix.arch }}-unknown-linux-gnu.zip.sha256sum", "./deno types > lib.deno.d.ts", ].join("\n"), }, @@ -784,8 +782,10 @@ const ci = { "--entitlements-xml-file=cli/entitlements.plist", "cd target/release", "zip -r deno-${{ matrix.arch }}-apple-darwin.zip deno", + "shasum -a 256 deno-${{ matrix.arch }}-apple-darwin.zip > deno-${{ matrix.arch }}-apple-darwin.zip.sha256sum", "strip denort", "zip -r denort-${{ matrix.arch }}-apple-darwin.zip denort", + "shasum -a 256 denort-${{ matrix.arch }}-apple-darwin.zip > denort-${{ matrix.arch }}-apple-darwin.zip.sha256sum", ] .join("\n"), }, @@ -800,7 +800,9 @@ const ci = { shell: "pwsh", run: [ "Compress-Archive -CompressionLevel Optimal -Force -Path target/release/deno.exe -DestinationPath target/release/deno-${{ matrix.arch }}-pc-windows-msvc.zip", + "Get-FileHash target/release/deno-${{ matrix.arch }}-pc-windows-msvc.zip -Algorithm SHA256 | Format-List > target/release/deno-${{ matrix.arch }}-pc-windows-msvc.zip.sha256sum", "Compress-Archive -CompressionLevel Optimal -Force -Path target/release/denort.exe -DestinationPath target/release/denort-${{ matrix.arch }}-pc-windows-msvc.zip", + "Get-FileHash target/release/denort-${{ matrix.arch }}-pc-windows-msvc.zip -Algorithm SHA256 | Format-List > target/release/denort-${{ matrix.arch }}-pc-windows-msvc.zip.sha256sum", ].join("\n"), }, { @@ -813,6 +815,7 @@ const ci = { ].join("\n"), run: [ 'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/canary/$(git rev-parse HEAD)/', + 'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.sha256sum gs://dl.deno.land/canary/$(git rev-parse HEAD)/', "echo ${{ github.sha }} > canary-latest.txt", 'gsutil -h "Cache-Control: no-cache" cp canary-latest.txt gs://dl.deno.land/canary-$(rustc -vV | sed -n "s|host: ||p")-latest.txt', ].join("\n"), @@ -994,8 +997,10 @@ const ci = { "github.repository == 'denoland/deno' &&", "startsWith(github.ref, 'refs/tags/')", ].join("\n"), - run: + run: [ 'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/', + 'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.sha256sum gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/', + ].join("\n"), }, { name: "Upload release to dl.deno.land (windows)", @@ -1009,8 +1014,10 @@ const ci = { env: { CLOUDSDK_PYTHON: "${{env.pythonLocation}}\\python.exe", }, - run: + run: [ 'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/', + 'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.sha256sum gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/', + ].join("\n"), }, { name: "Create release notes", @@ -1040,15 +1047,25 @@ const ci = { with: { files: [ "target/release/deno-x86_64-pc-windows-msvc.zip", + "target/release/deno-x86_64-pc-windows-msvc.zip.sha256sum", "target/release/denort-x86_64-pc-windows-msvc.zip", + "target/release/denort-x86_64-pc-windows-msvc.zip.sha256sum", "target/release/deno-x86_64-unknown-linux-gnu.zip", + "target/release/deno-x86_64-unknown-linux-gnu.zip.sha256sum", "target/release/denort-x86_64-unknown-linux-gnu.zip", + "target/release/denort-x86_64-unknown-linux-gnu.zip.sha256sum", "target/release/deno-x86_64-apple-darwin.zip", + "target/release/deno-x86_64-apple-darwin.zip.sha256sum", "target/release/denort-x86_64-apple-darwin.zip", + "target/release/denort-x86_64-apple-darwin.zip.sha256sum", "target/release/deno-aarch64-unknown-linux-gnu.zip", + "target/release/deno-aarch64-unknown-linux-gnu.zip.sha256sum", "target/release/denort-aarch64-unknown-linux-gnu.zip", + "target/release/denort-aarch64-unknown-linux-gnu.zip.sha256sum", "target/release/deno-aarch64-apple-darwin.zip", + "target/release/deno-aarch64-apple-darwin.zip.sha256sum", "target/release/denort-aarch64-apple-darwin.zip", + "target/release/denort-aarch64-apple-darwin.zip.sha256sum", "target/release/deno_src.tar.gz", "target/release/lib.deno.d.ts", ].join("\n"), @@ -1067,6 +1084,7 @@ const ci = { "./target", "!./target/*/gn_out", "!./target/*/*.zip", + "!./target/*/*.sha256sum", "!./target/*/*.tar.gz", ].join("\n"), key: prCacheKeyPrefix + "${{ github.sha }}", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 280eb7cb84c06e..dd48f1fb0d0077 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,18 +62,18 @@ jobs: profile: debug - os: macos arch: x86_64 - runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-22.04'' || ''macos-13'' }}' + runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || ''macos-13'' }}' job: test profile: release skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}' - os: macos arch: aarch64 - runner: macos-14 + runner: '${{ github.repository == ''denoland/deno'' && startsWith(github.ref, ''refs/tags/'') && ''self-hosted'' || ''macos-14'' }}' job: test profile: debug - os: macos arch: aarch64 - runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-22.04'' || ''macos-14'' }}' + runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || github.repository == ''denoland/deno'' && startsWith(github.ref, ''refs/tags/'') && ''self-hosted'' || ''macos-14'' }}' job: test profile: release skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}' @@ -84,33 +84,33 @@ jobs: profile: debug - os: windows arch: x86_64 - runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-22.04'' || github.repository == ''denoland/deno'' && ''windows-2022-xl'' || ''windows-2022'' }}' + runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'')) && ''ubuntu-24.04'' || github.repository == ''denoland/deno'' && ''windows-2022-xl'' || ''windows-2022'' }}' job: test profile: release skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'') }}' - os: linux arch: x86_64 - runner: '${{ github.repository == ''denoland/deno'' && ''ubuntu-22.04-xl'' || ''ubuntu-22.04'' }}' + runner: '${{ github.repository == ''denoland/deno'' && ''ubuntu-24.04-xl'' || ''ubuntu-24.04'' }}' job: test profile: release use_sysroot: true wpt: '${{ !startsWith(github.ref, ''refs/tags/'') }}' - os: linux arch: x86_64 - runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'' && !contains(github.event.pull_request.labels.*.name, ''ci-bench''))) && ''ubuntu-22.04'' || github.repository == ''denoland/deno'' && ''ubuntu-22.04-xl'' || ''ubuntu-22.04'' }}' + runner: '${{ (!contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'' && !contains(github.event.pull_request.labels.*.name, ''ci-bench''))) && ''ubuntu-24.04'' || github.repository == ''denoland/deno'' && ''ubuntu-24.04-xl'' || ''ubuntu-24.04'' }}' job: bench profile: release use_sysroot: true skip: '${{ !contains(github.event.pull_request.labels.*.name, ''ci-full'') && (github.event_name == ''pull_request'' && !contains(github.event.pull_request.labels.*.name, ''ci-bench'')) }}' - os: linux arch: x86_64 - runner: ubuntu-22.04 + runner: ubuntu-24.04 job: test profile: debug use_sysroot: true - os: linux arch: x86_64 - runner: ubuntu-22.04 + runner: ubuntu-24.04 job: lint profile: debug - os: linux @@ -178,7 +178,7 @@ jobs: if: '!(matrix.skip)' - if: '!(matrix.skip) && (matrix.job == ''lint'' || matrix.job == ''test'' || matrix.job == ''bench'')' name: Install Deno - uses: denoland/setup-deno@v1 + uses: denoland/setup-deno@v2 with: deno-version: v1.x - name: Install Python @@ -199,12 +199,6 @@ jobs: uses: actions/setup-node@v4 with: node-version: 18 - - name: Install protoc - uses: arduino/setup-protoc@v3 - with: - version: '21.12' - repo-token: '${{ secrets.GITHUB_TOKEN }}' - if: '!(matrix.skip)' - if: |- !(matrix.skip) && (matrix.profile == 'release' && matrix.job == 'test' && @@ -258,22 +252,22 @@ jobs: # to complete. sudo apt-get -qq remove --purge -y man-db > /dev/null 2> /dev/null # Remove older clang before we install - sudo apt-get -qq remove 'clang-12*' 'clang-13*' 'clang-14*' 'clang-15*' 'clang-16*' 'llvm-12*' 'llvm-13*' 'llvm-14*' 'llvm-15*' 'llvm-16*' 'lld-12*' 'lld-13*' 'lld-14*' 'lld-15*' 'lld-16*' > /dev/null 2> /dev/null + sudo apt-get -qq remove 'clang-12*' 'clang-13*' 'clang-14*' 'clang-15*' 'clang-16*' 'clang-17*' 'clang-18*' 'llvm-12*' 'llvm-13*' 'llvm-14*' 'llvm-15*' 'llvm-16*' 'lld-12*' 'lld-13*' 'lld-14*' 'lld-15*' 'lld-16*' 'lld-17*' 'lld-18*' > /dev/null 2> /dev/null # Install clang-XXX, lld-XXX, and debootstrap. - echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" | - sudo dd of=/etc/apt/sources.list.d/llvm-toolchain-jammy-18.list + echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-19 main" | + sudo dd of=/etc/apt/sources.list.d/llvm-toolchain-jammy-19.list curl https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor | sudo dd of=/etc/apt/trusted.gpg.d/llvm-snapshot.gpg sudo apt-get update # this was unreliable sometimes, so try again if it fails - sudo apt-get install --no-install-recommends clang-18 lld-18 clang-tools-18 clang-format-18 clang-tidy-18 || echo 'Failed. Trying again.' && sudo apt-get clean && sudo apt-get update && sudo apt-get install --no-install-recommends clang-18 lld-18 clang-tools-18 clang-format-18 clang-tidy-18 + sudo apt-get install --no-install-recommends clang-19 lld-19 clang-tools-19 clang-format-19 clang-tidy-19 || echo 'Failed. Trying again.' && sudo apt-get clean && sudo apt-get update && sudo apt-get install --no-install-recommends clang-19 lld-19 clang-tools-19 clang-format-19 clang-tidy-19 # Fix alternatives (yes '' | sudo update-alternatives --force --all) > /dev/null 2> /dev/null || true echo "Decompressing sysroot..." - wget -q https://github.com/denoland/deno_sysroot_build/releases/download/sysroot-20240528/sysroot-`uname -m`.tar.xz -O /tmp/sysroot.tar.xz + wget -q https://github.com/denoland/deno_sysroot_build/releases/download/sysroot-20241030/sysroot-`uname -m`.tar.xz -O /tmp/sysroot.tar.xz cd / xzcat /tmp/sysroot.tar.xz | sudo tar -x sudo mount --rbind /dev /sysroot/dev @@ -305,8 +299,8 @@ jobs: CARGO_PROFILE_RELEASE_LTO=false RUSTFLAGS<<__1 -C linker-plugin-lto=true - -C linker=clang-18 - -C link-arg=-fuse-ld=lld-18 + -C linker=clang-19 + -C link-arg=-fuse-ld=lld-19 -C link-arg=-ldl -C link-arg=-Wl,--allow-shlib-undefined -C link-arg=-Wl,--thinlto-cache-dir=$(pwd)/target/release/lto-cache @@ -316,8 +310,8 @@ jobs: __1 RUSTDOCFLAGS<<__1 -C linker-plugin-lto=true - -C linker=clang-18 - -C link-arg=-fuse-ld=lld-18 + -C linker=clang-19 + -C link-arg=-fuse-ld=lld-19 -C link-arg=-ldl -C link-arg=-Wl,--allow-shlib-undefined -C link-arg=-Wl,--thinlto-cache-dir=$(pwd)/target/release/lto-cache @@ -325,7 +319,7 @@ jobs: --cfg tokio_unstable $RUSTFLAGS __1 - CC=/usr/bin/clang-18 + CC=/usr/bin/clang-19 CFLAGS=-flto=thin $CFLAGS " > $GITHUB_ENV - name: Remove macOS cURL --ipv4 flag @@ -367,8 +361,8 @@ jobs: path: |- ~/.cargo/registry/index ~/.cargo/registry/cache - key: '15-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}' - restore-keys: '15-cargo-home-${{ matrix.os }}-${{ matrix.arch }}' + key: '28-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}' + restore-keys: '28-cargo-home-${{ matrix.os }}-${{ matrix.arch }}' if: '!(matrix.skip)' - name: Restore cache build output (PR) uses: actions/cache/restore@v4 @@ -381,7 +375,7 @@ jobs: !./target/*/*.zip !./target/*/*.tar.gz key: never_saved - restore-keys: '15-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-' + restore-keys: '28-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-' - name: Apply and update mtime cache if: '!(matrix.skip) && (!startsWith(github.ref, ''refs/tags/''))' uses: ./.github/mtime_cache @@ -449,8 +443,10 @@ jobs: run: |- cd target/release zip -r deno-${{ matrix.arch }}-unknown-linux-gnu.zip deno + shasum -a 256 deno-${{ matrix.arch }}-unknown-linux-gnu.zip > deno-${{ matrix.arch }}-unknown-linux-gnu.zip.sha256sum strip denort zip -r denort-${{ matrix.arch }}-unknown-linux-gnu.zip denort + shasum -a 256 denort-${{ matrix.arch }}-unknown-linux-gnu.zip > denort-${{ matrix.arch }}-unknown-linux-gnu.zip.sha256sum ./deno types > lib.deno.d.ts - name: Pre-release (mac) if: |- @@ -466,8 +462,10 @@ jobs: rcodesign sign target/release/deno --code-signature-flags=runtime --p12-password="$APPLE_CODESIGN_PASSWORD" --p12-file=<(echo $APPLE_CODESIGN_KEY | base64 -d) --entitlements-xml-file=cli/entitlements.plist cd target/release zip -r deno-${{ matrix.arch }}-apple-darwin.zip deno + shasum -a 256 deno-${{ matrix.arch }}-apple-darwin.zip > deno-${{ matrix.arch }}-apple-darwin.zip.sha256sum strip denort zip -r denort-${{ matrix.arch }}-apple-darwin.zip denort + shasum -a 256 denort-${{ matrix.arch }}-apple-darwin.zip > denort-${{ matrix.arch }}-apple-darwin.zip.sha256sum - name: Pre-release (windows) if: |- !(matrix.skip) && (matrix.os == 'windows' && @@ -477,7 +475,9 @@ jobs: shell: pwsh run: |- Compress-Archive -CompressionLevel Optimal -Force -Path target/release/deno.exe -DestinationPath target/release/deno-${{ matrix.arch }}-pc-windows-msvc.zip + Get-FileHash target/release/deno-${{ matrix.arch }}-pc-windows-msvc.zip -Algorithm SHA256 | Format-List > target/release/deno-${{ matrix.arch }}-pc-windows-msvc.zip.sha256sum Compress-Archive -CompressionLevel Optimal -Force -Path target/release/denort.exe -DestinationPath target/release/denort-${{ matrix.arch }}-pc-windows-msvc.zip + Get-FileHash target/release/denort-${{ matrix.arch }}-pc-windows-msvc.zip -Algorithm SHA256 | Format-List > target/release/denort-${{ matrix.arch }}-pc-windows-msvc.zip.sha256sum - name: Upload canary to dl.deno.land if: |- !(matrix.skip) && (matrix.job == 'test' && @@ -486,6 +486,7 @@ jobs: github.ref == 'refs/heads/main') run: |- gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/canary/$(git rev-parse HEAD)/ + gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.sha256sum gs://dl.deno.land/canary/$(git rev-parse HEAD)/ echo ${{ github.sha }} > canary-latest.txt gsutil -h "Cache-Control: no-cache" cp canary-latest.txt gs://dl.deno.land/canary-$(rustc -vV | sed -n "s|host: ||p")-latest.txt - name: Autobahn testsuite @@ -615,7 +616,9 @@ jobs: matrix.profile == 'release' && github.repository == 'denoland/deno' && startsWith(github.ref, 'refs/tags/')) - run: 'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/' + run: |- + gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/ + gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.sha256sum gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/ - name: Upload release to dl.deno.land (windows) if: |- !(matrix.skip) && (matrix.os == 'windows' && @@ -625,7 +628,9 @@ jobs: startsWith(github.ref, 'refs/tags/')) env: CLOUDSDK_PYTHON: '${{env.pythonLocation}}\python.exe' - run: 'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/' + run: |- + gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/ + gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.sha256sum gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/ - name: Create release notes if: |- !(matrix.skip) && (matrix.job == 'test' && @@ -647,15 +652,25 @@ jobs: with: files: |- target/release/deno-x86_64-pc-windows-msvc.zip + target/release/deno-x86_64-pc-windows-msvc.zip.sha256sum target/release/denort-x86_64-pc-windows-msvc.zip + target/release/denort-x86_64-pc-windows-msvc.zip.sha256sum target/release/deno-x86_64-unknown-linux-gnu.zip + target/release/deno-x86_64-unknown-linux-gnu.zip.sha256sum target/release/denort-x86_64-unknown-linux-gnu.zip + target/release/denort-x86_64-unknown-linux-gnu.zip.sha256sum target/release/deno-x86_64-apple-darwin.zip + target/release/deno-x86_64-apple-darwin.zip.sha256sum target/release/denort-x86_64-apple-darwin.zip + target/release/denort-x86_64-apple-darwin.zip.sha256sum target/release/deno-aarch64-unknown-linux-gnu.zip + target/release/deno-aarch64-unknown-linux-gnu.zip.sha256sum target/release/denort-aarch64-unknown-linux-gnu.zip + target/release/denort-aarch64-unknown-linux-gnu.zip.sha256sum target/release/deno-aarch64-apple-darwin.zip + target/release/deno-aarch64-apple-darwin.zip.sha256sum target/release/denort-aarch64-apple-darwin.zip + target/release/denort-aarch64-apple-darwin.zip.sha256sum target/release/deno_src.tar.gz target/release/lib.deno.d.ts body_path: target/release/release-notes.md @@ -668,11 +683,12 @@ jobs: ./target !./target/*/gn_out !./target/*/*.zip + !./target/*/*.sha256sum !./target/*/*.tar.gz - key: '15-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' + key: '28-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' publish-canary: name: publish canary - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: - build if: github.repository == 'denoland/deno' && github.ref == 'refs/heads/main' diff --git a/.github/workflows/post_publish.yml b/.github/workflows/post_publish.yml index c0db0906df414f..dd80b3637a4d0c 100644 --- a/.github/workflows/post_publish.yml +++ b/.github/workflows/post_publish.yml @@ -7,7 +7,7 @@ on: jobs: update-dl-version: name: update dl.deno.land version - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: github.repository == 'denoland/deno' steps: - name: Authenticate with Google Cloud diff --git a/.github/workflows/promote_to_release.yml b/.github/workflows/promote_to_release.yml index 3dc15dc73014f7..79fefa6d6c94c5 100644 --- a/.github/workflows/promote_to_release.yml +++ b/.github/workflows/promote_to_release.yml @@ -40,7 +40,7 @@ jobs: project_id: denoland - name: Install deno - uses: denoland/setup-deno@v1 + uses: denoland/setup-deno@v2 with: deno-version: v1.x diff --git a/.github/workflows/start_release.yml b/.github/workflows/start_release.yml index 392551afbe9182..40a44bb61aba64 100644 --- a/.github/workflows/start_release.yml +++ b/.github/workflows/start_release.yml @@ -16,7 +16,7 @@ on: jobs: build: name: start release - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 30 env: @@ -34,7 +34,7 @@ jobs: uses: actions/checkout@v4 - name: Install deno - uses: denoland/setup-deno@v1 + uses: denoland/setup-deno@v2 with: deno-version: v1.x diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml index 733abbb0246376..9038fe0d2295e9 100644 --- a/.github/workflows/version_bump.yml +++ b/.github/workflows/version_bump.yml @@ -16,7 +16,7 @@ on: jobs: build: name: version bump - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 90 env: @@ -39,7 +39,7 @@ jobs: - uses: dsherret/rust-toolchain-file@v1 - name: Install deno - uses: denoland/setup-deno@v1 + uses: denoland/setup-deno@v2 with: deno-version: v1.x diff --git a/.github/workflows/wpt_epoch.yml b/.github/workflows/wpt_epoch.yml index c3c6277b935652..1d86ed5557e55e 100644 --- a/.github/workflows/wpt_epoch.yml +++ b/.github/workflows/wpt_epoch.yml @@ -20,7 +20,7 @@ jobs: fail-fast: false matrix: deno-version: [v1.x, canary] - os: [ubuntu-22.04-xl] + os: [ubuntu-24.04-xl] steps: - name: Clone repository @@ -30,7 +30,7 @@ jobs: persist-credentials: false - name: Setup Deno - uses: denoland/setup-deno@v1 + uses: denoland/setup-deno@v2 with: deno-version: ${{ matrix.deno-version }} diff --git a/Cargo.lock b/Cargo.lock index bec2ec12cc0282..5e27d1a8011083 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -128,19 +128,6 @@ version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" -[[package]] -name = "ammonia" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e6d1c7838db705c9b756557ee27c384ce695a1c51a6fe528784cb1c6840170" -dependencies = [ - "html5ever", - "maplit", - "once_cell", - "tendril", - "url", -] - [[package]] name = "android_system_properties" version = "0.1.5" @@ -280,7 +267,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -316,7 +303,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -327,7 +314,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -351,7 +338,7 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -360,6 +347,59 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "axum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "az" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b7e4c2464d97fe331d41de9d5db0def0a96f4d823b8b32a2efd503578988973" + [[package]] name = "backtrace" version = "0.3.73" @@ -450,25 +490,22 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.69.4" +version = "0.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" dependencies = [ "bitflags 2.6.0", "cexpr", "clang-sys", - "itertools 0.10.5", - "lazy_static", - "lazycell", + "itertools 0.13.0", "log", - "prettyplease 0.2.17", + "prettyplease", "proc-macro2", "quote", "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.72", - "which 4.4.2", + "syn 2.0.87", ] [[package]] @@ -546,6 +583,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "boxed_error" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69aae56aaf59d1994b902ed5c0c79024012bdc2426741def75a635999a030e7e" +dependencies = [ + "quote", + "syn 2.0.87", +] + [[package]] name = "brotli" version = "6.0.0" @@ -631,12 +678,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.10" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9e8aabfac534be767c909e0690571677d49f41bd8465ae876fe043d52ba5292" +checksum = "066fce287b1d4eafef758e89e09d724a24808a9196fe9756b8ca90e86d0719a2" dependencies = [ "jobserver", "libc", + "once_cell", ] [[package]] @@ -688,7 +736,7 @@ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", - "libloading 0.8.3", + "libloading 0.8.5", ] [[package]] @@ -757,11 +805,13 @@ dependencies = [ "fastwebsockets", "file_test_runner", "flaky_test", + "hickory-client", + "hickory-server", "http 1.1.0", "http-body-util", "hyper 1.4.1", "hyper-util", - "nix 0.26.2", + "nix", "once_cell", "os_pipe", "pretty_assertions", @@ -770,8 +820,6 @@ dependencies = [ "serde", "test_server", "tokio", - "trust-dns-client", - "trust-dns-server", "url", "uuid", "zeromq", @@ -840,9 +888,9 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "comrak" -version = "0.26.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "395ab67843c57df5a4ee29d610740828dbc928cc64ecf0f2a1d5cd0e98e107a9" +checksum = "d8c32ff8b21372fab0e9ecc4e42536055702dc5faa418362bffd1544f9d12637" dependencies = [ "caseless", "derive_builder", @@ -1040,7 +1088,7 @@ dependencies = [ "cpufeatures", "curve25519-dalek-derive", "digest", - "fiat-crypto", + "fiat-crypto 0.2.7", "rustc_version 0.4.0", "subtle", "zeroize", @@ -1054,7 +1102,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1064,7 +1112,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b28bfe653d79bd16c77f659305b195b82bb5ce0c0eb2a4846b82ddbd77586813" dependencies = [ "bitflags 2.6.0", - "libloading 0.8.3", + "libloading 0.8.5", "winapi", ] @@ -1089,7 +1137,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1100,7 +1148,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1110,7 +1158,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core", @@ -1146,11 +1194,10 @@ dependencies = [ [[package]] name = "deno" -version = "2.0.0-rc.5" +version = "2.1.2" dependencies = [ "anstream", "async-trait", - "base32", "base64 0.21.7", "bincode", "bytes", @@ -1174,11 +1221,15 @@ dependencies = [ "deno_lockfile", "deno_npm", "deno_package_json", + "deno_path_util", + "deno_resolver", "deno_runtime", "deno_semver", "deno_task_shell", + "deno_telemetry", "deno_terminal 0.2.0", "deno_tower_lsp", + "dhat", "dissimilar", "dotenvy", "dprint-plugin-json", @@ -1186,7 +1237,6 @@ dependencies = [ "dprint-plugin-markdown", "dprint-plugin-typescript", "env_logger", - "eszip", "fancy-regex", "faster-hex", "flate2", @@ -1198,7 +1248,7 @@ dependencies = [ "http-body-util", "hyper-util", "import_map", - "indexmap", + "indexmap 2.3.0", "jsonc-parser", "junction", "lazy-regex", @@ -1211,8 +1261,7 @@ dependencies = [ "markup_fmt", "memmem", "monch", - "napi_sym", - "nix 0.26.2", + "nix", "node_resolver", "notify", "once_cell", @@ -1220,7 +1269,7 @@ dependencies = [ "p256", "pathdiff", "percent-encoding", - "phf 0.11.2", + "phf", "pretty_assertions", "pretty_yaml", "quick-junit", @@ -1236,6 +1285,7 @@ dependencies = [ "sha2", "shell-escape", "spki", + "sqlformat", "strsim", "tar", "tempfile", @@ -1253,7 +1303,6 @@ dependencies = [ "which 4.4.2", "winapi", "winres", - "yoke", "zeromq", "zip", "zstd", @@ -1267,14 +1316,14 @@ checksum = "05d59a1cfd445fd86f63616127a434aabca000e03d963b01b03ce813520565b9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "deno_ast" -version = "0.42.0" +version = "0.43.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b08d11d9e4086b00d3428650e31153cf5896586411763cb88a6423ce5b18791" +checksum = "48d00b724e06d2081a141ec1155756a0b465d413d8e2a7515221f61d482eb2ee" dependencies = [ "base64 0.21.7", "deno_media_type", @@ -1316,7 +1365,7 @@ dependencies = [ [[package]] name = "deno_bench_util" -version = "0.162.0" +version = "0.174.0" dependencies = [ "bencher", "deno_core", @@ -1325,34 +1374,38 @@ dependencies = [ [[package]] name = "deno_broadcast_channel" -version = "0.162.0" +version = "0.174.0" dependencies = [ "async-trait", "deno_core", + "thiserror", "tokio", "uuid", ] [[package]] name = "deno_cache" -version = "0.100.0" +version = "0.112.0" dependencies = [ "async-trait", "deno_core", "rusqlite", "serde", "sha2", + "thiserror", "tokio", ] [[package]] name = "deno_cache_dir" -version = "0.11.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6df43311cb7703fa3242c282823a850e4c8d0c06b9527d8209b55bd695452ea5" +checksum = "08c1f52170cd7715f8006da54cde1444863a0d6fbd9c11d037a737db2dec8e22" dependencies = [ + "base32", "deno_media_type", - "indexmap", + "deno_path_util", + "indexmap 2.3.0", "log", "once_cell", "parking_lot", @@ -1365,31 +1418,33 @@ dependencies = [ [[package]] name = "deno_canvas" -version = "0.37.0" +version = "0.49.0" dependencies = [ "deno_core", "deno_webgpu", "image", "serde", + "thiserror", ] [[package]] name = "deno_config" -version = "0.35.0" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "105864a9e0a7fbc22f1106784b2d263f402f157be1c3e1a9905f53d182700c9f" +checksum = "38fb809500238be2b10eee42944a47b3ac38974e1edbb47f73afcfca7df143bf" dependencies = [ "anyhow", "deno_package_json", + "deno_path_util", "deno_semver", "glob", "ignore", "import_map", - "indexmap", + "indexmap 2.3.0", "jsonc-parser", "log", "percent-encoding", - "phf 0.11.2", + "phf", "serde", "serde_json", "thiserror", @@ -1398,18 +1453,19 @@ dependencies = [ [[package]] name = "deno_console" -version = "0.168.0" +version = "0.180.0" dependencies = [ "deno_core", ] [[package]] name = "deno_core" -version = "0.311.0" +version = "0.323.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e09bd55da542fa1fde753aff617c355b5d782e763ab2a19e4371a56d7844cac" +checksum = "a781bcfe1b5211b8497f45bf5b3dba73036b8d5d1533c1f05d26ccf0afb25a78" dependencies = [ "anyhow", + "az", "bincode", "bit-set", "bit-vec", @@ -1419,8 +1475,9 @@ dependencies = [ "deno_ops", "deno_unsync", "futures", + "indexmap 2.3.0", "libc", - "memoffset 0.9.1", + "memoffset", "parking_lot", "percent-encoding", "pin-project", @@ -1433,29 +1490,31 @@ dependencies = [ "tokio", "url", "v8", + "wasm_dep_analyzer", ] [[package]] name = "deno_core_icudata" -version = "0.0.73" +version = "0.74.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13951ea98c0a4c372f162d669193b4c9d991512de9f2381dd161027f34b26b1" +checksum = "fe4dccb6147bb3f3ba0c7a48e993bfeb999d2c2e47a81badee80e2b370c8d695" [[package]] name = "deno_cron" -version = "0.48.0" +version = "0.60.0" dependencies = [ "anyhow", "async-trait", "chrono", "deno_core", "saffron", + "thiserror", "tokio", ] [[package]] name = "deno_crypto" -version = "0.182.0" +version = "0.194.0" dependencies = [ "aes", "aes-gcm", @@ -1467,6 +1526,7 @@ dependencies = [ "curve25519-dalek", "deno_core", "deno_web", + "ed448-goldilocks", "elliptic-curve", "num-traits", "once_cell", @@ -1483,39 +1543,44 @@ dependencies = [ "sha2", "signature", "spki", + "thiserror", + "tokio", "uuid", "x25519-dalek", ] [[package]] name = "deno_doc" -version = "0.148.0" +version = "0.161.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "144fa07977ba9eeeb98bcd267b7f0a6f8033f0f1f20fd210e669b3c4f30cefa2" +checksum = "32d994915f85e873865fc341e592080a487b0a987d06177016b2d93fd62162f8" dependencies = [ - "ammonia", "anyhow", "cfg-if", "comrak", "deno_ast", "deno_graph", - "futures", + "deno_path_util", "handlebars", "html-escape", "import_map", - "indexmap", + "indexmap 2.3.0", "itoa", + "js-sys", "lazy_static", + "percent-encoding", "regex", "serde", + "serde-wasm-bindgen", "serde_json", - "syntect", "termcolor", + "url", + "wasm-bindgen", ] [[package]] name = "deno_fetch" -version = "0.192.0" +version = "0.204.0" dependencies = [ "base64 0.21.7", "bytes", @@ -1526,6 +1591,7 @@ dependencies = [ "dyn-clone", "error_reporter", "fast-socks5", + "hickory-resolver", "http 1.1.0", "http-body-util", "hyper 1.4.1", @@ -1536,6 +1602,7 @@ dependencies = [ "rustls-webpki", "serde", "serde_json", + "thiserror", "tokio", "tokio-rustls", "tokio-socks", @@ -1547,7 +1614,7 @@ dependencies = [ [[package]] name = "deno_ffi" -version = "0.155.0" +version = "0.167.0" dependencies = [ "deno_core", "deno_permissions", @@ -1556,37 +1623,43 @@ dependencies = [ "libffi", "libffi-sys", "log", + "num-bigint", "serde", "serde-value", "serde_json", + "thiserror", + "tokio", "winapi", ] [[package]] name = "deno_fs" -version = "0.78.0" +version = "0.90.0" dependencies = [ "async-trait", "base32", + "boxed_error", "deno_core", "deno_io", + "deno_path_util", "deno_permissions", "filetime", "junction", "libc", - "nix 0.26.2", + "nix", "rand", "rayon", "serde", + "thiserror", "winapi", "windows-sys 0.52.0", ] [[package]] name = "deno_graph" -version = "0.82.2" +version = "0.86.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9328b62ffc7e806f1c92fd7a22e4ff3046fcb53f2d46e3e1297482b2c4c2bb9d" +checksum = "4c3f4be49dad28e794ff4eeb2daaf7956c97f8557097ef6f9c3ff1292e0a5c28" dependencies = [ "anyhow", "async-trait", @@ -1597,7 +1670,7 @@ dependencies = [ "encoding_rs", "futures", "import_map", - "indexmap", + "indexmap 2.3.0", "log", "monch", "once_cell", @@ -1609,11 +1682,12 @@ dependencies = [ "thiserror", "twox-hash", "url", + "wasm_dep_analyzer", ] [[package]] name = "deno_http" -version = "0.166.0" +version = "0.178.0" dependencies = [ "async-compression", "async-trait", @@ -1638,7 +1712,7 @@ dependencies = [ "mime", "once_cell", "percent-encoding", - "phf 0.11.2", + "phf", "pin-project", "rand", "ring", @@ -1652,7 +1726,7 @@ dependencies = [ [[package]] name = "deno_io" -version = "0.78.0" +version = "0.90.0" dependencies = [ "async-trait", "deno_core", @@ -1673,15 +1747,17 @@ dependencies = [ [[package]] name = "deno_kv" -version = "0.76.0" +version = "0.88.0" dependencies = [ "anyhow", "async-trait", "base64 0.21.7", + "boxed_error", "bytes", "chrono", "deno_core", "deno_fetch", + "deno_path_util", "deno_permissions", "deno_tls", "denokv_proto", @@ -1697,14 +1773,15 @@ dependencies = [ "rand", "rusqlite", "serde", + "thiserror", "url", ] [[package]] name = "deno_lint" -version = "0.67.0" +version = "0.68.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "871b60e32bfb6c110cbb9b0688dbf048f81e5d347fe4ce5a42239263de9dd938" +checksum = "bb994e6d1b18223df0a756c7948143b35682941d615edffef60d5b38822f38ac" dependencies = [ "anyhow", "deno_ast", @@ -1712,7 +1789,7 @@ dependencies = [ "if_chain", "log", "once_cell", - "phf 0.11.2", + "phf", "regex", "serde", "serde_json", @@ -1732,9 +1809,9 @@ dependencies = [ [[package]] name = "deno_media_type" -version = "0.1.4" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8978229b82552bf8457a0125aa20863f023619cfc21ebb007b1e571d68fd85b" +checksum = "7fcf552fbdedbe81c89705349d7d2485c7051382b000dfddbdbf7fc25931cf83" dependencies = [ "data-url", "serde", @@ -1743,11 +1820,17 @@ dependencies = [ [[package]] name = "deno_napi" -version = "0.99.0" +version = "0.111.0" dependencies = [ "deno_core", "deno_permissions", + "libc", "libloading 0.7.4", + "libuv-sys-lite", + "log", + "napi_sym", + "thiserror", + "windows-sys 0.52.0", ] [[package]] @@ -1765,29 +1848,31 @@ dependencies = [ [[package]] name = "deno_net" -version = "0.160.0" +version = "0.172.0" dependencies = [ "deno_core", "deno_permissions", "deno_tls", + "hickory-proto", + "hickory-resolver", "pin-project", "rustls-tokio-stream", "serde", "socket2", + "thiserror", "tokio", - "trust-dns-proto", - "trust-dns-resolver", ] [[package]] name = "deno_node" -version = "0.105.0" +version = "0.117.0" dependencies = [ "aead-gcm-stream", "aes", "async-trait", "base64 0.21.7", "blake2", + "boxed_error", "brotli", "bytes", "cbc", @@ -1800,6 +1885,7 @@ dependencies = [ "deno_media_type", "deno_net", "deno_package_json", + "deno_path_util", "deno_permissions", "deno_whoami", "der", @@ -1818,8 +1904,8 @@ dependencies = [ "http-body-util", "hyper 1.4.1", "hyper-util", - "idna 0.3.0", - "indexmap", + "idna 1.0.3", + "indexmap 2.3.0", "ipnetwork", "k256", "lazy-regex", @@ -1859,6 +1945,7 @@ dependencies = [ "stable_deref_trait", "thiserror", "tokio", + "tokio-eld", "url", "webpki-root-certs", "winapi", @@ -1870,9 +1957,9 @@ dependencies = [ [[package]] name = "deno_npm" -version = "0.25.2" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1809e2d77d8a06bc2800dc10c1d4acb664197e518e289a86e336411c1feba785" +checksum = "e6b4dc4a9f1cff63d5638e7d93042f24f46300d1cc77b86f3caaa699a7ddccf7" dependencies = [ "anyhow", "async-trait", @@ -1889,38 +1976,51 @@ dependencies = [ [[package]] name = "deno_ops" -version = "0.187.0" +version = "0.199.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e040fd4def8a67538fe38c9955fd970efc9f44284bd69d44f8992a456afd665d" +checksum = "a24a1f3e22029a57d3094b32070b8328eac793920b5a022027d360f085e6b245" dependencies = [ "proc-macro-rules", "proc-macro2", "quote", + "stringcase", "strum", "strum_macros", - "syn 2.0.72", + "syn 2.0.87", "thiserror", ] [[package]] name = "deno_package_json" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38cf6ea5cc98ea7ad58b0e84593773bea03fc0431071a296017bed4151e3dc1d" +checksum = "6cbc4c4d3eb0960b58e8f43f9fc2d3f620fcac9a03cd85203e08db5b04e83c1f" dependencies = [ "deno_semver", - "indexmap", + "indexmap 2.3.0", "serde", "serde_json", "thiserror", "url", ] +[[package]] +name = "deno_path_util" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff25f6e08e7a0214bbacdd6f7195c7f1ebcd850c87a624e4ff06326b68b42d99" +dependencies = [ + "percent-encoding", + "thiserror", + "url", +] + [[package]] name = "deno_permissions" -version = "0.28.0" +version = "0.40.0" dependencies = [ "deno_core", + "deno_path_util", "deno_terminal 0.2.0", "fqdn", "libc", @@ -1928,14 +2028,35 @@ dependencies = [ "once_cell", "percent-encoding", "serde", + "thiserror", "which 4.4.2", "winapi", ] +[[package]] +name = "deno_resolver" +version = "0.12.0" +dependencies = [ + "anyhow", + "base32", + "boxed_error", + "dashmap", + "deno_config", + "deno_media_type", + "deno_package_json", + "deno_path_util", + "deno_semver", + "node_resolver", + "test_server", + "thiserror", + "url", +] + [[package]] name = "deno_runtime" -version = "0.177.0" +version = "0.189.0" dependencies = [ + "color-print", "deno_ast", "deno_broadcast_channel", "deno_cache", @@ -1953,7 +2074,9 @@ dependencies = [ "deno_napi", "deno_net", "deno_node", + "deno_path_util", "deno_permissions", + "deno_telemetry", "deno_terminal 0.2.0", "deno_tls", "deno_url", @@ -1974,7 +2097,7 @@ dependencies = [ "libc", "log", "netif", - "nix 0.26.2", + "nix", "node_resolver", "notify", "ntapi", @@ -1982,10 +2105,13 @@ dependencies = [ "percent-encoding", "regex", "rustyline", + "same-file", "serde", "signal-hook", "signal-hook-registry", + "tempfile", "test_server", + "thiserror", "tokio", "tokio-metrics", "twox-hash", @@ -1997,9 +2123,9 @@ dependencies = [ [[package]] name = "deno_semver" -version = "0.5.13" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6657fecb9ac6a7a71f552c95e8cc492466a75f5660224577e2226bcf30db9768" +checksum = "c957c6a57c38b7dde2315df0da0ec228911e56a74f185b108a488d0401841a67" dependencies = [ "monch", "once_cell", @@ -2010,9 +2136,9 @@ dependencies = [ [[package]] name = "deno_task_shell" -version = "0.17.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd6413ffc1654cad015edb5c4ab574069acdc929a6efafed23bc947901bcff1a" +checksum = "4f444918f7102c1a5a143e9d57809e499fb4d365070519bf2e8bdb16d586af2a" dependencies = [ "anyhow", "futures", @@ -2025,6 +2151,27 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "deno_telemetry" +version = "0.2.0" +dependencies = [ + "async-trait", + "deno_core", + "http-body-util", + "hyper 1.4.1", + "hyper-util", + "log", + "once_cell", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", + "pin-project", + "serde", + "tokio", +] + [[package]] name = "deno_terminal" version = "0.1.1" @@ -2047,7 +2194,7 @@ dependencies = [ [[package]] name = "deno_tls" -version = "0.155.0" +version = "0.167.0" dependencies = [ "deno_core", "deno_native_certs", @@ -2056,6 +2203,7 @@ dependencies = [ "rustls-tokio-stream", "rustls-webpki", "serde", + "thiserror", "tokio", "webpki-roots", ] @@ -2095,18 +2243,19 @@ dependencies = [ [[package]] name = "deno_url" -version = "0.168.0" +version = "0.180.0" dependencies = [ "deno_bench_util", "deno_console", "deno_core", "deno_webidl", + "thiserror", "urlpattern", ] [[package]] name = "deno_web" -version = "0.199.0" +version = "0.211.0" dependencies = [ "async-trait", "base64-simd 0.8.0", @@ -2121,17 +2270,19 @@ dependencies = [ "flate2", "futures", "serde", + "thiserror", "tokio", "uuid", ] [[package]] name = "deno_webgpu" -version = "0.135.0" +version = "0.147.0" dependencies = [ "deno_core", "raw-window-handle", "serde", + "thiserror", "tokio", "wgpu-core", "wgpu-types", @@ -2139,7 +2290,7 @@ dependencies = [ [[package]] name = "deno_webidl" -version = "0.168.0" +version = "0.180.0" dependencies = [ "deno_bench_util", "deno_core", @@ -2147,7 +2298,7 @@ dependencies = [ [[package]] name = "deno_websocket" -version = "0.173.0" +version = "0.185.0" dependencies = [ "bytes", "deno_core", @@ -2163,16 +2314,18 @@ dependencies = [ "once_cell", "rustls-tokio-stream", "serde", + "thiserror", "tokio", ] [[package]] name = "deno_webstorage" -version = "0.163.0" +version = "0.175.0" dependencies = [ "deno_core", "deno_web", "rusqlite", + "thiserror", ] [[package]] @@ -2187,9 +2340,9 @@ dependencies = [ [[package]] name = "denokv_proto" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f629c77d2bf59e2e2a07fd7b14bfffae352a3813fbdcb801e72205741fb7625c" +checksum = "f7ba1f99ed11a9c11e868a8521b1f71a7e1aba785d7f42ea9ecbdc01146c89ec" dependencies = [ "anyhow", "async-trait", @@ -2203,9 +2356,9 @@ dependencies = [ [[package]] name = "denokv_remote" -version = "0.8.1" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d57717b5123e8d1ec5f52973a67f98e3621274d362d18b245038967b402082df" +checksum = "08ed833073189e8f6d03155fe3b05a024e75e29d8a28a4c2e9ec3b5c925e727b" dependencies = [ "anyhow", "async-stream", @@ -2228,9 +2381,9 @@ dependencies = [ [[package]] name = "denokv_sqlite" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c4f5719e2bf698ec4f39fe29d91b62ff06a4b4a37ee481ffb8658d140fed986" +checksum = "9b790f01d1302d53a0c3cbd27de88a06b3abd64ec8ab8673924e490541c7c713" dependencies = [ "anyhow", "async-stream", @@ -2285,7 +2438,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -2305,7 +2458,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -2326,7 +2479,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -2336,7 +2489,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" dependencies = [ "derive_builder_core", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -2358,6 +2511,22 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6e854126756c496b8c81dec88f9a706b15b875c5849d4097a3854476b9fdf94" +[[package]] +name = "dhat" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98cd11d84628e233de0ce467de10b8633f4ddaecafadefc86e13b84b8739b827" +dependencies = [ + "backtrace", + "lazy_static", + "mintex", + "parking_lot", + "rustc-hash 1.1.0", + "serde", + "serde_json", + "thousands", +] + [[package]] name = "diff" version = "0.1.13" @@ -2405,7 +2574,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -2434,7 +2603,7 @@ checksum = "f2b99bf03862d7f545ebc28ddd33a665b50865f4dfd84031a393823879bd4c54" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -2460,8 +2629,8 @@ checksum = "f3ab0dd2bedc109d25f0d21afb09b7d329f6c6fa83b095daf31d2d967e091548" dependencies = [ "anyhow", "bumpalo", - "hashbrown", - "indexmap", + "hashbrown 0.14.5", + "indexmap 2.3.0", "rustc-hash 1.1.0", "serde", "unicode-width", @@ -2479,9 +2648,9 @@ dependencies = [ [[package]] name = "dprint-plugin-json" -version = "0.19.3" +version = "0.19.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19f4a9f2f548b2098b8ec597d7bb40af133b6e9a3187c1d3c4caa101b8c93c3" +checksum = "57f91e594559b450b7c5d6a0ba9f3f9fe951c1ea371168f7c95973da3fdbd85a" dependencies = [ "anyhow", "dprint-core", @@ -2493,9 +2662,9 @@ dependencies = [ [[package]] name = "dprint-plugin-jupyter" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c67b0e54b552a4775c221b44ed33be918c400bd8041d1f044f947fbb01025cc0" +checksum = "d0d20684e37b3824e2bc917cfcb14e2cdf88398eef507335d839cbd78172bfee" dependencies = [ "anyhow", "dprint-core", @@ -2521,9 +2690,9 @@ dependencies = [ [[package]] name = "dprint-plugin-typescript" -version = "0.93.0" +version = "0.93.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9308d98b923b7c0335c2ee1560199e3f2321b1be82803107b4ba4ed5dac46cc" +checksum = "3ff29fd136541e59d51946f0d2d353fefc886776f61a799ebfb5838b06cef13b" dependencies = [ "anyhow", "deno_ast", @@ -2551,15 +2720,6 @@ dependencies = [ "text_lines", ] -[[package]] -name = "drain" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d105028bd2b5dfcb33318fd79a445001ead36004dd8dffef1bdd7e493d8bc1e" -dependencies = [ - "tokio", -] - [[package]] name = "dsa" version = "0.6.3" @@ -2657,6 +2817,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ed448-goldilocks" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06924531e9e90130842b012e447f85bdaf9161bc8a0f8092be8cb70b01ebe092" +dependencies = [ + "fiat-crypto 0.1.20", + "hex", + "subtle", + "zeroize", +] + [[package]] name = "editpe" version = "0.1.0" @@ -2664,7 +2836,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48cede2bb1b07dd598d269f973792c43e0cd92686d3b452bd6e01d7a8eb01211" dependencies = [ "debug-ignore", - "indexmap", + "indexmap 2.3.0", "log", "thiserror", "zerocopy", @@ -2730,7 +2902,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -2795,29 +2967,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31ae425815400e5ed474178a7a22e275a9687086a12ca63ec793ff292d8fdae8" -[[package]] -name = "eszip" -version = "0.78.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0546f00d41dbc6e90b50e922759c02559a897e59b683369c3a13519cd5108b6" -dependencies = [ - "anyhow", - "async-trait", - "base64 0.21.7", - "deno_ast", - "deno_graph", - "deno_npm", - "deno_semver", - "futures", - "hashlink 0.8.4", - "indexmap", - "serde", - "serde_json", - "sha2", - "thiserror", - "url", -] - [[package]] name = "fallible-iterator" version = "0.3.0" @@ -2919,6 +3068,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "fiat-crypto" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e825f6987101665dea6ec934c09ec6d721de7bc1bf92248e1d5810c8cd636b77" + [[package]] name = "fiat-crypto" version = "0.2.7" @@ -2927,9 +3082,9 @@ checksum = "c007b1ae3abe1cb6f85a16305acd418b7ca6343b953633fee2b76d8f108b830f" [[package]] name = "file_test_runner" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b23dcc1b671771c6f59fdace6da685735c925f859733e8fd07fba6cae6462a" +checksum = "cf50901549edf2241e33d1715aec0575adc5510a09724877a1e0afe7ffafb0fb" dependencies = [ "anyhow", "crossbeam-channel", @@ -3031,7 +3186,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -3063,7 +3218,7 @@ checksum = "fdc9cc75639b041067353b9bce2450d6847e547276c6fbe4487d7407980e07db" dependencies = [ "proc-macro2", "swc_macros_common", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -3102,16 +3257,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" -[[package]] -name = "futf" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" -dependencies = [ - "mac", - "new_debug_unreachable", -] - [[package]] name = "futures" version = "0.3.30" @@ -3168,7 +3313,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -3328,7 +3473,7 @@ checksum = "9c08c1f623a8d0b722b8b99f821eb0ba672a1618f0d3b16ddbee1cedd2dd8557" dependencies = [ "bitflags 2.6.0", "gpu-descriptor-types", - "hashbrown", + "hashbrown 0.14.5", ] [[package]] @@ -3372,7 +3517,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap", + "indexmap 2.3.0", "slab", "tokio", "tokio-util", @@ -3391,7 +3536,7 @@ dependencies = [ "futures-sink", "futures-util", "http 1.1.0", - "indexmap", + "indexmap 2.3.0", "slab", "tokio", "tokio-util", @@ -3404,15 +3549,15 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8588661a8607108a5ca69cab034063441a0413a0b041c13618a7dd348021ef6f" dependencies = [ - "hashbrown", + "hashbrown 0.14.5", "serde", ] [[package]] name = "handlebars" -version = "5.1.2" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d08485b96a0e6393e9e4d1b8d48cf74ad6c063cd905eb33f42c1ce3f0377539b" +checksum = "ce25b617d1375ef96eeb920ae717e3da34a02fc979fe632c75128350f9e1f74a" dependencies = [ "heck 0.5.0", "log", @@ -3423,6 +3568,12 @@ dependencies = [ "thiserror", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -3435,20 +3586,25 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.8.4" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" dependencies = [ - "hashbrown", + "hashbrown 0.14.5", ] [[package]] -name = "hashlink" -version = "0.9.1" +name = "hdrhistogram" +version = "7.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" dependencies = [ - "hashbrown", + "base64 0.21.7", + "byteorder", + "crossbeam-channel", + "flate2", + "nom 7.1.3", + "num-traits", ] [[package]] @@ -3482,21 +3638,107 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" [[package]] -name = "hkdf" -version = "0.12.4" +name = "hickory-client" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +checksum = "bab9683b08d8f8957a857b0236455d80e1886eaa8c6178af556aa7871fb61b55" dependencies = [ - "hmac", + "cfg-if", + "data-encoding", + "futures-channel", + "futures-util", + "hickory-proto", + "once_cell", + "radix_trie", + "rand", + "thiserror", + "tokio", + "tracing", ] [[package]] -name = "hmac" -version = "0.12.1" +name = "hickory-proto" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +checksum = "07698b8420e2f0d6447a436ba999ec85d8fbf2a398bbd737b82cac4a2e96e512" dependencies = [ - "digest", + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna 0.4.0", + "ipnet", + "once_cell", + "rand", + "serde", + "thiserror", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28757f23aa75c98f254cf0405e6d8c25b831b32921b050a66692427679b1f243" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-proto", + "ipconfig", + "lru-cache", + "once_cell", + "parking_lot", + "rand", + "resolv-conf", + "serde", + "smallvec", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "hickory-server" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9be0e43c556b9b3fdb6c7c71a9a32153a2275d02419e3de809e520bfcfe40c37" +dependencies = [ + "async-trait", + "bytes", + "cfg-if", + "enum-as-inner", + "futures-util", + "hickory-proto", + "serde", + "thiserror", + "time", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", ] [[package]] @@ -3525,10 +3767,10 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a9de2bdef6354361892492bab5e316b2d78a0ee9971db4d36da9b1eb0e11999" dependencies = [ - "hashbrown", + "hashbrown 0.14.5", "new_debug_unreachable", "once_cell", - "phf 0.11.2", + "phf", "rustc-hash 1.1.0", "triomphe", ] @@ -3542,20 +3784,6 @@ dependencies = [ "utf8-width", ] -[[package]] -name = "html5ever" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7" -dependencies = [ - "log", - "mac", - "markup5ever", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "http" version = "0.2.12" @@ -3693,6 +3921,19 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "hyper-timeout" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +dependencies = [ + "hyper 1.4.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.7" @@ -3713,6 +3954,124 @@ dependencies = [ "tracing", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -3721,9 +4080,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -3731,12 +4090,23 @@ dependencies = [ [[package]] name = "idna" -version = "0.4.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] @@ -3781,7 +4151,7 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "351a787decc56f38d65d16d32687265045d6d6a4531b4a0e1b649def3590354e" dependencies = [ - "indexmap", + "indexmap 2.3.0", "log", "percent-encoding", "serde", @@ -3790,6 +4160,16 @@ dependencies = [ "url", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.3.0" @@ -3797,7 +4177,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.14.5", "serde", ] @@ -3876,7 +4256,7 @@ dependencies = [ "Inflector", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -3956,9 +4336,9 @@ dependencies = [ [[package]] name = "jsonc-parser" -version = "0.23.0" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7725c320caac8c21d8228c1d055af27a995d371f78cc763073d3e068323641b5" +checksum = "b558af6b49fd918e970471374e7a798b2c9bbcda624a210ffa3901ee5614bc8e" dependencies = [ "serde_json", ] @@ -3973,6 +4353,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "jupyter-serde" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd71aa17c4fa65e6d7536ab2728881a41f8feb2ee5841c2240516c3c3d65d8b3" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "thiserror", + "uuid", +] + [[package]] name = "k256" version = "0.13.3" @@ -4003,7 +4396,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76" dependencies = [ "libc", - "libloading 0.8.3", + "libloading 0.8.5", "pkg-config", ] @@ -4053,7 +4446,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -4065,76 +4458,6 @@ dependencies = [ "spin", ] -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - -[[package]] -name = "lexical-core" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" -dependencies = [ - "lexical-parse-float", - "lexical-parse-integer", - "lexical-util", - "lexical-write-float", - "lexical-write-integer", -] - -[[package]] -name = "lexical-parse-float" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" -dependencies = [ - "lexical-parse-integer", - "lexical-util", - "static_assertions", -] - -[[package]] -name = "lexical-parse-integer" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" -dependencies = [ - "lexical-util", - "static_assertions", -] - -[[package]] -name = "lexical-util" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" -dependencies = [ - "static_assertions", -] - -[[package]] -name = "lexical-write-float" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" -dependencies = [ - "lexical-util", - "lexical-write-integer", - "static_assertions", -] - -[[package]] -name = "lexical-write-integer" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" -dependencies = [ - "lexical-util", - "static_assertions", -] - [[package]] name = "libc" version = "0.2.153" @@ -4172,9 +4495,9 @@ dependencies = [ [[package]] name = "libloading" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" +checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", "windows-targets 0.52.4", @@ -4198,9 +4521,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.30.1" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +checksum = "6b694a822684ddb75df4d657029161431bcb4a85c1856952f845b76912bc6fec" dependencies = [ "cc", "pkg-config", @@ -4209,9 +4532,9 @@ dependencies = [ [[package]] name = "libsui" -version = "0.3.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e39af24eff8df7c8b9980ef56a1a1f4d2e77b34b2d5c0529f108c53ae96a7a" +checksum = "89795977654ad6250d6c0915411b622bac22f9efb4f852af94b2e00964cab832" dependencies = [ "editpe", "libc", @@ -4220,11 +4543,21 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "libuv-sys-lite" +version = "1.48.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8dfd1a173826d193e3b955e07c22765829890f62c677a59c4a410cb4f47c01" +dependencies = [ + "bindgen", + "libloading 0.8.5", +] + [[package]] name = "libz-sys" -version = "1.1.16" +version = "1.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e143b5e666b2695d28f6bca6497720813f699c9602dd7f5cac91008b8ada7f9" +checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" dependencies = [ "cc", "cmake", @@ -4245,6 +4578,12 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "litrs" version = "0.4.1" @@ -4292,12 +4631,6 @@ dependencies = [ "serde_repr", ] -[[package]] -name = "mac" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" - [[package]] name = "malloc_buf" version = "0.0.6" @@ -4309,9 +4642,9 @@ dependencies = [ [[package]] name = "malva" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "484beda6e5d775ed06a8ec0fce79e51d39f49d834ed2a29da3f437079321804f" +checksum = "1c67b97ed99f56b86fa3c010843441f1fcdb71884bab96b8551bb3d1e7c6d529" dependencies = [ "aho-corasick", "itertools 0.13.0", @@ -4320,31 +4653,11 @@ dependencies = [ "tiny_pretty", ] -[[package]] -name = "maplit" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" - -[[package]] -name = "markup5ever" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016" -dependencies = [ - "log", - "phf 0.10.1", - "phf_codegen", - "string_cache", - "string_cache_codegen", - "tendril", -] - [[package]] name = "markup_fmt" -version = "0.13.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74fc137a4a591720176339bf7e857586a48ff35c0caee7ad6cf709327901232c" +checksum = "f303c36143671ac6c54112eb5aa95649b169dae783fdb6ead2c0e88b408c425c" dependencies = [ "aho-corasick", "css_dataset", @@ -4359,6 +4672,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "md-5" version = "0.10.6" @@ -4399,15 +4718,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a64a92489e2744ce060c349162be1c5f33c6969234104dbd99ddb5feb08b8c15" -[[package]] -name = "memoffset" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.9.1" @@ -4454,6 +4764,12 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "mintex" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bec4598fddb13cc7b528819e697852653252b760f1228b7642679bf2ff2cd07" + [[package]] name = "mio" version = "0.8.11" @@ -4489,7 +4805,7 @@ dependencies = [ "bitflags 2.6.0", "codespan-reporting", "hexf-parse", - "indexmap", + "indexmap 2.3.0", "log", "num-traits", "rustc-hash 1.1.0", @@ -4517,12 +4833,12 @@ dependencies = [ [[package]] name = "napi_sym" -version = "0.98.0" +version = "0.110.0" dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -4559,20 +4875,6 @@ dependencies = [ "smallvec", ] -[[package]] -name = "nix" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" -dependencies = [ - "bitflags 1.3.2", - "cfg-if", - "libc", - "memoffset 0.7.1", - "pin-utils", - "static_assertions", -] - [[package]] name = "nix" version = "0.27.1" @@ -4586,12 +4888,14 @@ dependencies = [ [[package]] name = "node_resolver" -version = "0.7.0" +version = "0.19.0" dependencies = [ "anyhow", "async-trait", + "boxed_error", "deno_media_type", "deno_package_json", + "deno_path_util", "futures", "lazy-regex", "once_cell", @@ -4761,28 +5065,6 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" -[[package]] -name = "onig" -version = "6.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" -dependencies = [ - "bitflags 1.3.2", - "libc", - "once_cell", - "onig_sys", -] - -[[package]] -name = "onig_sys" -version = "69.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "opaque-debug" version = "0.3.1" @@ -4806,6 +5088,93 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "opentelemetry" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f3cebff57f7dbd1255b44d8bddc2cebeb0ea677dbaa2e25a3070a91b318f660" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", +] + +[[package]] +name = "opentelemetry-http" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a8a7f5f6ba7c1b286c2fbca0454eaba116f63bbe69ed250b642d36fbb04d80" +dependencies = [ + "async-trait", + "bytes", + "http 1.1.0", + "opentelemetry", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" +dependencies = [ + "async-trait", + "futures-core", + "http 1.1.0", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "serde_json", + "thiserror", + "tokio", + "tonic", + "tracing", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6" +dependencies = [ + "hex", + "opentelemetry", + "opentelemetry_sdk", + "prost", + "serde", + "tonic", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52" + +[[package]] +name = "opentelemetry_sdk" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b742c1cae4693792cc564e58d75a2a0ba29421a34a85b50da92efa89ecb2bc" +dependencies = [ + "async-trait", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "once_cell", + "opentelemetry", + "percent-encoding", + "rand", + "serde_json", + "thiserror", + "tracing", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -5010,7 +5379,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5031,16 +5400,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap", -] - -[[package]] -name = "phf" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" -dependencies = [ - "phf_shared 0.10.0", + "indexmap 2.3.0", ] [[package]] @@ -5050,27 +5410,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ "phf_macros", - "phf_shared 0.11.2", -] - -[[package]] -name = "phf_codegen" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd" -dependencies = [ - "phf_generator 0.10.0", - "phf_shared 0.10.0", -] - -[[package]] -name = "phf_generator" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" -dependencies = [ - "phf_shared 0.10.0", - "rand", + "phf_shared", ] [[package]] @@ -5079,7 +5419,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ - "phf_shared 0.11.2", + "phf_shared", "rand", ] @@ -5089,20 +5429,11 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" dependencies = [ - "phf_generator 0.11.2", - "phf_shared 0.11.2", + "phf_generator", + "phf_shared", "proc-macro2", "quote", - "syn 2.0.72", -] - -[[package]] -name = "phf_shared" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" -dependencies = [ - "siphasher", + "syn 2.0.87", ] [[package]] @@ -5131,7 +5462,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5227,12 +5558,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" -[[package]] -name = "precomputed-hash" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" - [[package]] name = "pretty_assertions" version = "1.4.0" @@ -5254,16 +5579,6 @@ dependencies = [ "yaml_parser", ] -[[package]] -name = "prettyplease" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" -dependencies = [ - "proc-macro2", - "syn 1.0.109", -] - [[package]] name = "prettyplease" version = "0.2.17" @@ -5271,7 +5586,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d3928fb5db768cb86f891ff014f0144589297e3c6a1aba6ed7cecfdace270c7" dependencies = [ "proc-macro2", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5315,7 +5630,7 @@ checksum = "07c277e4e643ef00c1233393c673f655e3672cf7eb3ba08a00bdd0ea59139b5f" dependencies = [ "proc-macro-rules-macros", "proc-macro2", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5327,7 +5642,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5347,9 +5662,9 @@ checksum = "43d84d1d7a6ac92673717f9f6d1518374ef257669c24ebc5ac25d5033828be58" [[package]] name = "prost" -version = "0.11.9" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" dependencies = [ "bytes", "prost-derive", @@ -5357,44 +5672,43 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.9" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +checksum = "0c1318b19085f08681016926435853bbf7858f9c082d0999b80550ff5d9abe15" dependencies = [ "bytes", - "heck 0.4.1", - "itertools 0.10.5", - "lazy_static", + "heck 0.5.0", + "itertools 0.13.0", "log", "multimap", + "once_cell", "petgraph", - "prettyplease 0.1.25", + "prettyplease", "prost", "prost-types", "regex", - "syn 1.0.109", + "syn 2.0.87", "tempfile", - "which 4.4.2", ] [[package]] name = "prost-derive" -version = "0.11.9" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.87", ] [[package]] name = "prost-types" -version = "0.11.9" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670" dependencies = [ "prost", ] @@ -5452,7 +5766,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1a341ae463320e9f8f34adda49c8a85d81d4e8f34cce4397fb0350481552224" dependencies = [ "chrono", - "indexmap", + "indexmap 2.3.0", "quick-xml", "strip-ansi-escapes", "thiserror", @@ -5517,9 +5831,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -5564,7 +5878,7 @@ checksum = "4fdb50eb5bf734fa5a770680a61876a6ec77b99c1e0e52d1f18ad6ebfa85759f" dependencies = [ "heck 0.5.0", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5666,7 +5980,7 @@ checksum = "5fddb4f8d99b0a2ebafc65a87a69a7b9875e4b1ae1f00db265d300ef7f28bccc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -5814,8 +6128,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a58fa8a7ccff2aec4f39cc45bf5f985cec7125ab271cf681c279fd00192b49" dependencies = [ "countme", - "hashbrown", - "memoffset 0.9.1", + "hashbrown 0.14.5", + "memoffset", "rustc-hash 1.1.0", "text-size", ] @@ -5842,9 +6156,9 @@ dependencies = [ [[package]] name = "runtimelib" -version = "0.14.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3d817764e3971867351e6103955b17d808f5330e9ef63aaaaab55bf8c664c1" +checksum = "fe23ba9967355bbb1be2fb9a8e51bd239ffdf9c791fad5a9b765122ee2bde2e4" dependencies = [ "anyhow", "base64 0.22.1", @@ -5854,6 +6168,7 @@ dependencies = [ "dirs", "futures", "glob", + "jupyter-serde", "rand", "ring", "serde", @@ -5866,14 +6181,14 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.32.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +checksum = "1cdbe9230a57259b37f7257d0aff38b8c9dbda3513edba2105e59b130189d82f" dependencies = [ "bitflags 2.6.0", "fallible-iterator", "fallible-streaming-iterator", - "hashlink 0.9.1", + "hashlink", "libsqlite3-sys", "smallvec", ] @@ -6023,7 +6338,7 @@ dependencies = [ "libc", "log", "memchr", - "nix 0.27.1", + "nix", "radix_trie", "unicode-segmentation", "unicode-width", @@ -6193,6 +6508,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3b143e2833c57ab9ad3ea280d21fd34e285a42837aeb0ee301f4f41890fa00e" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_bytes" version = "0.11.14" @@ -6210,7 +6536,7 @@ checksum = "692d6f5ac90220161d6774db30c662202721e64aed9058d2c394f451261420c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -6219,7 +6545,7 @@ version = "1.0.122" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da" dependencies = [ - "indexmap", + "indexmap 2.3.0", "itoa", "memchr", "ryu", @@ -6234,16 +6560,7 @@ checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", -] - -[[package]] -name = "serde_spanned" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" -dependencies = [ - "serde", + "syn 2.0.87", ] [[package]] @@ -6260,9 +6577,9 @@ dependencies = [ [[package]] name = "serde_v8" -version = "0.220.0" +version = "0.232.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7a65d91d79acc82aa229aeb084f4a39bda269069bc1520df40f679495388e4" +checksum = "5c9feae92f7293fcc1a32a86be1a399859c0637e55dad8991d5258c43f7ff4d2" dependencies = [ "num-bigint", "serde", @@ -6386,13 +6703,12 @@ checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" [[package]] name = "simd-json" -version = "0.13.9" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0b84c23a1066e1d650ebc99aa8fb9f8ed0ab96fd36e2e836173c92fc9fb29bc" +checksum = "05f0b376aada35f30a0012f5790e50aed62f91804a0682669aefdbe81c7fcb91" dependencies = [ "getrandom", "halfbrown", - "lexical-core", "ref-cast", "serde", "serde_json", @@ -6539,6 +6855,18 @@ dependencies = [ "der", ] +[[package]] +name = "sqlformat" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c38684453189293372e6fffa3bed1015d20488ce4cc09a23de050fd7411e46" +dependencies = [ + "nom 7.1.3", + "once_cell", + "regex", + "unicode_categories", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -6564,32 +6892,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "string_cache" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" -dependencies = [ - "new_debug_unreachable", - "once_cell", - "parking_lot", - "phf_shared 0.10.0", - "precomputed-hash", - "serde", -] - -[[package]] -name = "string_cache_codegen" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" -dependencies = [ - "phf_generator 0.10.0", - "phf_shared 0.10.0", - "proc-macro2", - "quote", -] - [[package]] name = "string_enum" version = "0.4.4" @@ -6599,9 +6901,15 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.72", + "syn 2.0.87", ] +[[package]] +name = "stringcase" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04028eeb851ed08af6aba5caa29f2d59a13ed168cee4d6bd753aeefcf1d636b0" + [[package]] name = "strip-ansi-escapes" version = "0.2.0" @@ -6636,7 +6944,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -6652,7 +6960,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adc8bd3075d1c6964010333fae9ddcd91ad422a4f8eb8b3206a9b2b6afb4209e" dependencies = [ "bumpalo", - "hashbrown", + "hashbrown 0.14.5", "ptr_meta", "rustc-hash 1.1.0", "triomphe", @@ -6678,7 +6986,7 @@ checksum = "c77c112c218a09635d99a45802a81b4f341d6c28c81076aa2c29ba3bcd9151a9" dependencies = [ "anyhow", "crc", - "indexmap", + "indexmap 2.3.0", "is-macro", "once_cell", "parking_lot", @@ -6748,7 +7056,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4740e53eaf68b101203c1df0937d5161a29f3c13bceed0836ddfe245b72dd000" dependencies = [ "anyhow", - "indexmap", + "indexmap 2.3.0", "serde", "serde_json", "swc_cached", @@ -6764,7 +7072,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -6776,7 +7084,7 @@ dependencies = [ "bitflags 2.6.0", "is-macro", "num-bigint", - "phf 0.11.2", + "phf", "scoped-tls", "serde", "string_enum", @@ -6813,7 +7121,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -6840,7 +7148,7 @@ dependencies = [ "new_debug_unreachable", "num-bigint", "num-traits", - "phf 0.11.2", + "phf", "serde", "smallvec", "smartstring", @@ -6860,9 +7168,9 @@ checksum = "65f21494e75d0bd8ef42010b47cabab9caaed8f2207570e809f6f4eb51a710d1" dependencies = [ "better_scoped_tls", "bitflags 2.6.0", - "indexmap", + "indexmap 2.3.0", "once_cell", - "phf 0.11.2", + "phf", "rustc-hash 1.1.0", "serde", "smallvec", @@ -6898,7 +7206,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -6908,7 +7216,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98d8447ea20ef76958a8240feef95743702485a84331e6df5bdbe7e383c87838" dependencies = [ "dashmap", - "indexmap", + "indexmap 2.3.0", "once_cell", "petgraph", "rustc-hash 1.1.0", @@ -6953,7 +7261,7 @@ checksum = "76c76d8b9792ce51401d38da0fa62158d61f6d80d16d68fe5b03ce4bf5fba383" dependencies = [ "base64 0.21.7", "dashmap", - "indexmap", + "indexmap 2.3.0", "once_cell", "serde", "sha1", @@ -6993,7 +7301,7 @@ version = "0.134.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "029eec7dd485923a75b5a45befd04510288870250270292fc2c1b3a9e7547408" dependencies = [ - "indexmap", + "indexmap 2.3.0", "num_cpus", "once_cell", "rustc-hash 1.1.0", @@ -7029,7 +7337,7 @@ checksum = "63db0adcff29d220c3d151c5b25c0eabe7e32dd936212b84cdaa1392e3130497" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -7038,7 +7346,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357e2c97bb51431d65080f25b436bc4e2fc1a7f64a643bc21a8353e478dc799f" dependencies = [ - "indexmap", + "indexmap 2.3.0", "petgraph", "rustc-hash 1.1.0", "swc_common", @@ -7065,7 +7373,7 @@ checksum = "f486687bfb7b5c560868f69ed2d458b880cebc9babebcb67e49f31b55c5bf847" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -7088,7 +7396,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -7104,9 +7412,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.72" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -7139,27 +7447,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", -] - -[[package]] -name = "syntect" -version = "5.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874dcfa363995604333cf947ae9f751ca3af4522c60886774c4963943b4746b1" -dependencies = [ - "bincode", - "bitflags 1.3.2", - "flate2", - "fnv", - "once_cell", - "onig", - "regex-syntax", - "serde", - "serde_derive", - "serde_json", - "thiserror", - "walkdir", + "syn 2.0.87", ] [[package]] @@ -7191,17 +7479,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "tendril" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" -dependencies = [ - "futf", - "mac", - "utf-8", -] - [[package]] name = "termcolor" version = "1.4.1" @@ -7233,6 +7510,7 @@ dependencies = [ name = "test_napi" version = "0.1.0" dependencies = [ + "libuv-sys-lite", "napi-build", "napi-sys", "test_server", @@ -7249,6 +7527,7 @@ dependencies = [ "console_static_text", "deno_unsync", "denokv_proto", + "faster-hex", "fastwebsockets", "flate2", "futures", @@ -7263,7 +7542,7 @@ dependencies = [ "libc", "lsp-types", "monch", - "nix 0.26.2", + "nix", "once_cell", "os_pipe", "parking_lot", @@ -7304,24 +7583,30 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.61" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.61" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] +[[package]] +name = "thousands" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820" + [[package]] name = "thread_local" version = "1.1.8" @@ -7372,6 +7657,16 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -7406,6 +7701,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "tokio-eld" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9166030f05d6bc5642bdb8f8c2be31eb3c02cd465d662bcdc2df82d4aa41a584" +dependencies = [ + "hdrhistogram", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.2.0" @@ -7414,7 +7719,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -7454,9 +7759,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite", @@ -7474,7 +7779,7 @@ dependencies = [ "futures-io", "futures-sink", "futures-util", - "hashbrown", + "hashbrown 0.14.5", "pin-project-lite", "slab", "tokio", @@ -7491,37 +7796,33 @@ dependencies = [ ] [[package]] -name = "toml" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.19.15" +name = "tonic" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ - "indexmap", - "serde", - "serde_spanned", - "toml_datetime", - "winnow 0.5.40", + "async-stream", + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2 0.4.4", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.4.1", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", ] [[package]] @@ -7532,11 +7833,16 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", + "indexmap 1.9.3", "pin-project", "pin-project-lite", + "rand", + "slab", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -7591,7 +7897,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -7613,95 +7919,6 @@ dependencies = [ "stable_deref_trait", ] -[[package]] -name = "trust-dns-client" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14135e72c7e6d4c9b6902d4437881a8598f0145dbb2e3f86f92dbad845b61e63" -dependencies = [ - "cfg-if", - "data-encoding", - "futures-channel", - "futures-util", - "once_cell", - "radix_trie", - "rand", - "thiserror", - "tokio", - "tracing", - "trust-dns-proto", -] - -[[package]] -name = "trust-dns-proto" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3119112651c157f4488931a01e586aa459736e9d6046d3bd9105ffb69352d374" -dependencies = [ - "async-trait", - "cfg-if", - "data-encoding", - "enum-as-inner", - "futures-channel", - "futures-io", - "futures-util", - "idna 0.4.0", - "ipnet", - "once_cell", - "rand", - "serde", - "smallvec", - "thiserror", - "tinyvec", - "tokio", - "tracing", - "url", -] - -[[package]] -name = "trust-dns-resolver" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a3e6c3aff1718b3c73e395d1f35202ba2ffa847c6a62eea0db8fb4cfe30be6" -dependencies = [ - "cfg-if", - "futures-util", - "ipconfig", - "lru-cache", - "once_cell", - "parking_lot", - "rand", - "resolv-conf", - "serde", - "smallvec", - "thiserror", - "tokio", - "tracing", - "trust-dns-proto", -] - -[[package]] -name = "trust-dns-server" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c540f73c2b2ec2f6c54eabd0900e7aafb747a820224b742f556e8faabb461bc7" -dependencies = [ - "async-trait", - "bytes", - "cfg-if", - "drain", - "enum-as-inner", - "futures-executor", - "futures-util", - "serde", - "thiserror", - "time", - "tokio", - "toml 0.7.8", - "tracing", - "trust-dns-proto", -] - [[package]] name = "try-lock" version = "0.2.5" @@ -7862,12 +8079,12 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.4.1" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" dependencies = [ "form_urlencoded", - "idna 0.4.0", + "idna 1.0.3", "percent-encoding", "serde", ] @@ -7890,12 +8107,24 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + [[package]] name = "utf8-width" version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.1" @@ -7915,9 +8144,9 @@ dependencies = [ [[package]] name = "v8" -version = "0.106.0" +version = "130.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a381badc47c6f15acb5fe0b5b40234162349ed9d4e4fd7c83a7f5547c0fc69c5" +checksum = "c23b5c2caff00209b03a716609b275acae94b02dd3b63c4648e7232a84a8402f" dependencies = [ "bindgen", "bitflags 2.6.0", @@ -7938,7 +8167,7 @@ checksum = "97599c400fc79925922b58303e98fcb8fa88f573379a08ddb652e72cbd2e70f6" dependencies = [ "bitflags 2.6.0", "encoding_rs", - "indexmap", + "indexmap 2.3.0", "num-bigint", "serde", "thiserror", @@ -7947,9 +8176,9 @@ dependencies = [ [[package]] name = "value-trait" -version = "0.8.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad8db98c1e677797df21ba03fca7d3bf9bec3ca38db930954e4fe6e1ea27eb4" +checksum = "bcaa56177466248ba59d693a048c0959ddb67f1151b963f904306312548cf392" dependencies = [ "float-cmp", "halfbrown", @@ -8049,7 +8278,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", "wasm-bindgen-shared", ] @@ -8083,7 +8312,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -8107,6 +8336,15 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm_dep_analyzer" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f270206a91783fd90625c8bb0d8fbd459d0b1d1bf209b656f713f01ae7c04b8" +dependencies = [ + "thiserror", +] + [[package]] name = "web-sys" version = "0.3.69" @@ -8119,9 +8357,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "0.26.5" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d93b773107ba49bc84dd3b241e019c702d886fd5c457defe2ea8b1123a5dcd" +checksum = "e8c6dfa3ac045bc517de14c7b1384298de1dbd229d38e08e169d9ae8c170937c" dependencies = [ "rustls-pki-types", ] @@ -8147,7 +8385,7 @@ dependencies = [ "cfg_aliases", "codespan-reporting", "document-features", - "indexmap", + "indexmap 2.3.0", "log", "naga", "once_cell", @@ -8186,7 +8424,7 @@ dependencies = [ "js-sys", "khronos-egl", "libc", - "libloading 0.8.3", + "libloading 0.8.5", "log", "metal", "naga", @@ -8451,15 +8689,6 @@ version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" -[[package]] -name = "winnow" -version = "0.5.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" -dependencies = [ - "memchr", -] - [[package]] name = "winnow" version = "0.6.15" @@ -8495,7 +8724,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b68db261ef59e9e52806f688020631e987592bd83619edccda9c47d42cde4f6c" dependencies = [ - "toml 0.5.11", + "toml", ] [[package]] @@ -8504,6 +8733,18 @@ version = "0.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "wtf8" version = "0.1.0" @@ -8572,7 +8813,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a6a39b6b5ba0d02c910d05d7fbc366a4befb8901ea107dcde9c1c97acb8a366" dependencies = [ "rowan", - "winnow 0.6.15", + "winnow", ] [[package]] @@ -8601,7 +8842,7 @@ checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", "synstructure 0.13.1", ] @@ -8623,7 +8864,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -8643,7 +8884,7 @@ checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", "synstructure 0.13.1", ] @@ -8664,14 +8905,14 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "zeromq" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb0560d00172817b7f7c2265060783519c475702ae290b154115ca75e976d4d0" +checksum = "6a4528179201f6eecf211961a7d3276faa61554c82651ecc66387f68fc3004bd" dependencies = [ "async-trait", "asynchronous-codec", @@ -8694,6 +8935,28 @@ dependencies = [ "uuid", ] +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "zip" version = "2.1.6" @@ -8705,7 +8968,7 @@ dependencies = [ "crossbeam-utils", "displaydoc", "flate2", - "indexmap", + "indexmap 2.3.0", "memchr", "thiserror", ] diff --git a/Cargo.toml b/Cargo.toml index 81cbf8b4c6a449..1ca9fcb66b7172 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,6 @@ resolver = "2" members = [ "bench_util", "cli", - "cli/napi/sym", "ext/broadcast_channel", "ext/cache", "ext/canvas", @@ -19,15 +18,18 @@ members = [ "ext/io", "ext/kv", "ext/napi", + "ext/napi/sym", "ext/net", "ext/node", - "ext/node_resolver", + "ext/telemetry", "ext/url", "ext/web", "ext/webgpu", "ext/webidl", "ext/websocket", "ext/webstorage", + "resolvers/deno", + "resolvers/node", "runtime", "runtime/permissions", "tests", @@ -44,48 +46,55 @@ license = "MIT" repository = "https://github.com/denoland/deno" [workspace.dependencies] -deno_ast = { version = "=0.42.0", features = ["transpiling"] } -deno_core = { version = "0.311.0" } +deno_ast = { version = "=0.43.3", features = ["transpiling"] } +deno_core = { version = "0.323.0" } -deno_bench_util = { version = "0.162.0", path = "./bench_util" } +deno_bench_util = { version = "0.174.0", path = "./bench_util" } +deno_config = { version = "=0.39.2", features = ["workspace", "sync"] } deno_lockfile = "=0.23.1" -deno_media_type = { version = "0.1.4", features = ["module_specifier"] } -deno_permissions = { version = "0.28.0", path = "./runtime/permissions" } -deno_runtime = { version = "0.177.0", path = "./runtime" } -deno_semver = "=0.5.13" +deno_media_type = { version = "0.2.0", features = ["module_specifier"] } +deno_npm = "=0.25.4" +deno_path_util = "=0.2.1" +deno_permissions = { version = "0.40.0", path = "./runtime/permissions" } +deno_runtime = { version = "0.189.0", path = "./runtime" } +deno_semver = "=0.5.16" deno_terminal = "0.2.0" -napi_sym = { version = "0.98.0", path = "./cli/napi/sym" } +napi_sym = { version = "0.110.0", path = "./ext/napi/sym" } test_util = { package = "test_server", path = "./tests/util/server" } -denokv_proto = "0.8.1" -denokv_remote = "0.8.1" +denokv_proto = "0.8.4" +denokv_remote = "0.8.4" # denokv_sqlite brings in bundled sqlite if we don't disable the default features -denokv_sqlite = { default-features = false, version = "0.8.2" } +denokv_sqlite = { default-features = false, version = "0.8.4" } # exts -deno_broadcast_channel = { version = "0.162.0", path = "./ext/broadcast_channel" } -deno_cache = { version = "0.100.0", path = "./ext/cache" } -deno_canvas = { version = "0.37.0", path = "./ext/canvas" } -deno_console = { version = "0.168.0", path = "./ext/console" } -deno_cron = { version = "0.48.0", path = "./ext/cron" } -deno_crypto = { version = "0.182.0", path = "./ext/crypto" } -deno_fetch = { version = "0.192.0", path = "./ext/fetch" } -deno_ffi = { version = "0.155.0", path = "./ext/ffi" } -deno_fs = { version = "0.78.0", path = "./ext/fs" } -deno_http = { version = "0.166.0", path = "./ext/http" } -deno_io = { version = "0.78.0", path = "./ext/io" } -deno_kv = { version = "0.76.0", path = "./ext/kv" } -deno_napi = { version = "0.99.0", path = "./ext/napi" } -deno_net = { version = "0.160.0", path = "./ext/net" } -deno_node = { version = "0.105.0", path = "./ext/node" } -deno_tls = { version = "0.155.0", path = "./ext/tls" } -deno_url = { version = "0.168.0", path = "./ext/url" } -deno_web = { version = "0.199.0", path = "./ext/web" } -deno_webgpu = { version = "0.135.0", path = "./ext/webgpu" } -deno_webidl = { version = "0.168.0", path = "./ext/webidl" } -deno_websocket = { version = "0.173.0", path = "./ext/websocket" } -deno_webstorage = { version = "0.163.0", path = "./ext/webstorage" } -node_resolver = { version = "0.7.0", path = "./ext/node_resolver" } +deno_broadcast_channel = { version = "0.174.0", path = "./ext/broadcast_channel" } +deno_cache = { version = "0.112.0", path = "./ext/cache" } +deno_canvas = { version = "0.49.0", path = "./ext/canvas" } +deno_console = { version = "0.180.0", path = "./ext/console" } +deno_cron = { version = "0.60.0", path = "./ext/cron" } +deno_crypto = { version = "0.194.0", path = "./ext/crypto" } +deno_fetch = { version = "0.204.0", path = "./ext/fetch" } +deno_ffi = { version = "0.167.0", path = "./ext/ffi" } +deno_fs = { version = "0.90.0", path = "./ext/fs" } +deno_http = { version = "0.178.0", path = "./ext/http" } +deno_io = { version = "0.90.0", path = "./ext/io" } +deno_kv = { version = "0.88.0", path = "./ext/kv" } +deno_napi = { version = "0.111.0", path = "./ext/napi" } +deno_net = { version = "0.172.0", path = "./ext/net" } +deno_node = { version = "0.117.0", path = "./ext/node" } +deno_telemetry = { version = "0.2.0", path = "./ext/telemetry" } +deno_tls = { version = "0.167.0", path = "./ext/tls" } +deno_url = { version = "0.180.0", path = "./ext/url" } +deno_web = { version = "0.211.0", path = "./ext/web" } +deno_webgpu = { version = "0.147.0", path = "./ext/webgpu" } +deno_webidl = { version = "0.180.0", path = "./ext/webidl" } +deno_websocket = { version = "0.185.0", path = "./ext/websocket" } +deno_webstorage = { version = "0.175.0", path = "./ext/webstorage" } + +# resolvers +deno_resolver = { version = "0.12.0", path = "./resolvers/deno" } +node_resolver = { version = "0.19.0", path = "./resolvers/node" } aes = "=0.8.3" anyhow = "1.0.57" @@ -93,6 +102,7 @@ async-trait = "0.1.73" base32 = "=0.5.1" base64 = "0.21.7" bencher = "0.1" +boxed_error = "0.2.2" brotli = "6.0.0" bytes = "1.4.0" cache_control = "=0.2.0" @@ -100,11 +110,13 @@ cbc = { version = "=0.1.2", features = ["alloc"] } # Note: Do not use the "clock" feature of chrono, as it links us to CoreFoundation on macOS. # Instead use util::time::utc_now() chrono = { version = "0.4", default-features = false, features = ["std", "serde"] } +color-print = "0.3.5" console_static_text = "=0.8.1" +dashmap = "5.5.3" data-encoding = "2.3.3" data-url = "=0.3.0" -deno_cache_dir = "=0.11.1" -deno_package_json = { version = "=0.1.1", default-features = false } +deno_cache_dir = "=0.13.2" +deno_package_json = { version = "0.1.2", default-features = false } dlopen2 = "0.6.1" ecb = "=0.1.2" elliptic-curve = { version = "0.13.4", features = ["alloc", "arithmetic", "ecdh", "std", "pem", "jwk"] } @@ -118,6 +130,7 @@ fs3 = "0.5.0" futures = "0.3.21" glob = "0.3.1" h2 = "0.4.4" +hickory-resolver = { version = "0.24", features = ["tokio-runtime", "serde-config"] } http = "1.0" http-body = "1.0" http-body-util = "0.1.2" @@ -129,11 +142,11 @@ hyper-util = { version = "=0.1.7", features = ["tokio", "client", "client-legacy hyper_v014 = { package = "hyper", version = "0.14.26", features = ["runtime", "http1"] } indexmap = { version = "2", features = ["serde"] } ipnet = "2.3" -jsonc-parser = { version = "=0.23.0", features = ["serde"] } +jsonc-parser = { version = "=0.26.2", features = ["serde"] } lazy-regex = "3" libc = "0.2.126" -libz-sys = { version = "1.1", default-features = false } -log = "0.4.20" +libz-sys = { version = "1.1.20", default-features = false } +log = { version = "0.4.20", features = ["kv"] } lsp-types = "=0.97.0" # used by tower-lsp and "proposed" feature is unstable in patch releases memmem = "0.1.1" monch = "=0.5.0" @@ -149,8 +162,8 @@ percent-encoding = "2.3.0" phf = { version = "0.11", features = ["macros"] } pin-project = "1.0.11" # don't pin because they yank crates from cargo pretty_assertions = "=1.4.0" -prost = "0.11" -prost-build = "0.11" +prost = "0.13" +prost-build = "0.13" rand = "=0.8.5" regex = "^1.7.0" reqwest = { version = "=0.12.5", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli", "socks", "json", "http2"] } # pinned because of https://github.com/seanmonstar/reqwest/pull/1955 @@ -179,7 +192,7 @@ tar = "=0.4.40" tempfile = "3.4.0" termcolor = "1.1.3" thiserror = "1.0.61" -tokio = { version = "=1.36.0", features = ["full"] } +tokio = { version = "1.36.0", features = ["full"] } tokio-metrics = { version = "0.3.0", features = ["rt"] } tokio-rustls = { version = "0.26.0", default-features = false, features = ["ring", "tls12"] } tokio-socks = "0.5.1" @@ -189,16 +202,21 @@ tower-http = { version = "0.6.1", features = ["decompression-br", "decompression tower-lsp = { package = "deno_tower_lsp", version = "0.1.0", features = ["proposed"] } tower-service = "0.3.2" twox-hash = "=1.6.3" -# Upgrading past 2.4.1 may cause WPT failures -url = { version = "< 2.5.0", features = ["serde", "expose_internals"] } +url = { version = "2.5", features = ["serde", "expose_internals"] } uuid = { version = "1.3.0", features = ["v4"] } webpki-root-certs = "0.26.5" webpki-roots = "0.26" which = "4.2.5" yoke = { version = "0.7.4", features = ["derive"] } -zeromq = { version = "=0.4.0", default-features = false, features = ["tcp-transport", "tokio-runtime"] } +zeromq = { version = "=0.4.1", default-features = false, features = ["tcp-transport", "tokio-runtime"] } zstd = "=0.12.4" +opentelemetry = "0.27.0" +opentelemetry-http = "0.27.0" +opentelemetry-otlp = { version = "0.27.0", features = ["logs", "http-proto", "http-json"] } +opentelemetry-semantic-conventions = { version = "0.27.0", features = ["semconv_experimental"] } +opentelemetry_sdk = "0.27.0" + # crypto hkdf = "0.12.3" rsa = { version = "0.9.3", default-features = false, features = ["std", "pem", "hazmat"] } # hazmat needed for PrehashSigner in ext/node @@ -213,15 +231,14 @@ quote = "1" syn = { version = "2", features = ["full", "extra-traits"] } # unix -nix = "=0.26.2" +nix = "=0.27.1" # windows deps junction = "=0.2.0" winapi = "=0.3.9" -windows-sys = { version = "0.52.0", features = ["Win32_Foundation", "Win32_Media", "Win32_Storage_FileSystem", "Win32_System_IO", "Win32_System_WindowsProgramming", "Wdk", "Wdk_System", "Wdk_System_SystemInformation", "Win32_System_Pipes", "Wdk_Storage_FileSystem", "Win32_System_Registry"] } +windows-sys = { version = "0.52.0", features = ["Win32_Foundation", "Win32_Media", "Win32_Storage_FileSystem", "Win32_System_IO", "Win32_System_WindowsProgramming", "Wdk", "Wdk_System", "Wdk_System_SystemInformation", "Win32_Security", "Win32_System_Pipes", "Wdk_Storage_FileSystem", "Win32_System_Registry", "Win32_System_Kernel"] } winres = "=0.1.12" -# NB: the `bench` and `release` profiles must remain EXACTLY the same. [profile.release] codegen-units = 1 incremental = true @@ -239,13 +256,6 @@ inherits = "release" codegen-units = 128 lto = "thin" -# NB: the `bench` and `release` profiles must remain EXACTLY the same. -[profile.bench] -codegen-units = 1 -incremental = true -lto = true -opt-level = 'z' # Optimize for size - # Key generation is too slow on `debug` [profile.dev.package.num-bigint-dig] opt-level = 3 @@ -254,80 +264,6 @@ opt-level = 3 [profile.dev.package.v8] opt-level = 1 -# Optimize these packages for performance. -# NB: the `bench` and `release` profiles must remain EXACTLY the same. -[profile.bench.package.async-compression] -opt-level = 3 -[profile.bench.package.base64-simd] -opt-level = 3 -[profile.bench.package.brotli] -opt-level = 3 -[profile.bench.package.brotli-decompressor] -opt-level = 3 -[profile.bench.package.bytes] -opt-level = 3 -[profile.bench.package.deno_bench_util] -opt-level = 3 -[profile.bench.package.deno_broadcast_channel] -opt-level = 3 -[profile.bench.package.deno_core] -opt-level = 3 -[profile.bench.package.deno_crypto] -opt-level = 3 -[profile.bench.package.deno_fetch] -opt-level = 3 -[profile.bench.package.deno_ffi] -opt-level = 3 -[profile.bench.package.deno_http] -opt-level = 3 -[profile.bench.package.deno_napi] -opt-level = 3 -[profile.bench.package.deno_net] -opt-level = 3 -[profile.bench.package.deno_node] -opt-level = 3 -[profile.bench.package.deno_runtime] -opt-level = 3 -[profile.bench.package.deno_tls] -opt-level = 3 -[profile.bench.package.deno_url] -opt-level = 3 -[profile.bench.package.deno_web] -opt-level = 3 -[profile.bench.package.deno_websocket] -opt-level = 3 -[profile.bench.package.fastwebsockets] -opt-level = 3 -[profile.bench.package.flate2] -opt-level = 3 -[profile.bench.package.futures-util] -opt-level = 3 -[profile.bench.package.hyper] -opt-level = 3 -[profile.bench.package.miniz_oxide] -opt-level = 3 -[profile.bench.package.num-bigint-dig] -opt-level = 3 -[profile.bench.package.rand] -opt-level = 3 -[profile.bench.package.serde] -opt-level = 3 -[profile.bench.package.serde_v8] -opt-level = 3 -[profile.bench.package.test_napi] -opt-level = 3 -[profile.bench.package.tokio] -opt-level = 3 -[profile.bench.package.url] -opt-level = 3 -[profile.bench.package.v8] -opt-level = 3 -[profile.bench.package.zstd] -opt-level = 3 -[profile.bench.package.zstd-sys] -opt-level = 3 - -# NB: the `bench` and `release` profiles must remain EXACTLY the same. [profile.release.package.async-compression] opt-level = 3 [profile.release.package.base64-simd] @@ -386,6 +322,8 @@ opt-level = 3 opt-level = 3 [profile.release.package.serde_v8] opt-level = 3 +[profile.release.package.libsui] +opt-level = 3 [profile.release.package.test_napi] opt-level = 3 [profile.release.package.tokio] diff --git a/README.md b/README.md index 102319f4f2b82d..19d4fa8a120129 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,12 @@ brew install deno choco install deno ``` +[WinGet](https://winstall.app/apps/DenoLand.Deno) (Windows): + +```powershell +winget install --id=DenoLand.Deno +``` + ### Build and install from source Complete instructions for building Deno from source can be found in the manual diff --git a/Releases.md b/Releases.md index a6e4d84e7bb8c3..6f9083ffde8c16 100644 --- a/Releases.md +++ b/Releases.md @@ -6,6 +6,662 @@ https://github.com/denoland/deno/releases We also have one-line install commands at: https://github.com/denoland/deno_install +### 2.1.2 / 2024.11.28 + +- feat(unstable): Instrument Deno.serve (#26964) +- feat(unstable): Instrument fetch (#27057) +- feat(unstable): repurpose `--unstable-detect-cjs` to attempt loading more + modules as cjs (#27094) +- fix(check): support jsdoc `@import` tag (#26991) +- fix(compile): correct buffered reading of assets and files (#27008) +- fix(compile): do not error embedding same symlink via multiple methods + (#27015) +- fix(compile): handle TypeScript file included as asset (#27032) +- fix(ext/fetch): don't throw when `bodyUsed` inspect after upgrade (#27088) +- fix(ext/node): `tls.connect` socket upgrades (#27125) +- fix(ext/node): add `fs.promises.fstat` and `FileHandle#stat` (#26719) +- fix(ext/webgpu): normalize limits to number (#27072) +- fix(ext/webgpu): use correct variable name (#27108) +- fix(ext/websocket): don't throw exception when sending to closed socket + (#26932) +- fix(fmt): return `None` if sql fmt result is the same (#27014) +- fix(info): resolve bare specifier pointing to workspace member (#27020) +- fix(init): always force managed node modules (#27047) +- fix(init): support scoped npm packages (#27128) +- fix(install): don't re-set up node_modules if running lifecycle script + (#26984) +- fix(lsp): remove stray debug output (#27010) +- fix(lsp): support task object notation for tasks request (#27076) +- fix(lsp): wasm file import completions (#27018) +- fix(node): correct resolution of dynamic import of esm from cjs (#27071) +- fix(node/fs): add missing stat path argument validation (#27086) +- fix(node/fs): missing uv error context for readFile (#27011) +- fix(node/http): casing ignored in ServerResponse.hasHeader() (#27105) +- fix(node/timers): error when passing id to clearTimeout/clearInterval (#27130) +- fix(runtime/ops): Fix watchfs remove event (#27041) +- fix(streams): reject `string` in `ReadableStream.from` type (#25116) +- fix(task): handle carriage return in task description (#27099) +- fix(task): handle multiline descriptions properly (#27069) +- fix(task): strip ansi codes and control chars when printing tasks (#27100) +- fix(tools/doc): HTML resolve main entrypoint from config file (#27103) +- fix: support bun specifiers in JSR publish (#24588) +- fix: support non-function exports in Wasm modules (#26992) +- perf(compile): read embedded files as static references when UTF-8 and reading + as strings (#27033) +- perf(ext/webstorage): use object wrap for `Storage` (#26931) + +### 2.1.1 / 2024.11.21 + +- docs(add): clarification to add command (#26968) +- docs(doc): fix typo in doc subcommand help output (#26321) +- fix(node): regression where ts files were sometimes resolved instead of js + (#26971) +- fix(task): ensure root config always looks up dependencies in root (#26959) +- fix(watch): don't panic if there's no path provided (#26972) +- fix: Buffer global in --unstable-node-globals (#26973) + +### 2.1.0 / 2024.11.21 + +- feat(cli): add `--unstable-node-globals` flag (#26617) +- feat(cli): support multiple env file argument (#26527) +- feat(compile): ability to embed directory in executable (#26939) +- feat(compile): ability to embed local data files (#26934) +- feat(ext/fetch): Make fetch client parameters configurable (#26909) +- feat(ext/fetch): allow embedders to use `hickory_dns_resolver` instead of + default `GaiResolver` (#26740) +- feat(ext/fs): add ctime to Deno.stats and use it in node compat layer (#24801) +- feat(ext/http): Make http server parameters configurable (#26785) +- feat(ext/node): perf_hooks.monitorEventLoopDelay() (#26905) +- feat(fetch): accept async iterables for body (#26882) +- feat(fmt): support SQL (#26750) +- feat(info): show location for Web Cache (#26205) +- feat(init): add --npm flag to initialize npm projects (#26896) +- feat(jupyter): Add `Deno.jupyter.image` API (#26284) +- feat(lint): Add checked files list to the JSON output(#26936) +- feat(lsp): auto-imports with @deno-types directives (#26821) +- feat(node): stabilize detecting if CJS via `"type": "commonjs"` in a + package.json (#26439) +- feat(permission): support suffix wildcards in `--allow-env` flag (#25255) +- feat(publish): add `--set-version ` flag (#26141) +- feat(runtime): remove public OTEL trace API (#26854) +- feat(task): add --eval flag (#26943) +- feat(task): dependencies (#26467) +- feat(task): support object notation, remove support for JSDocs (#26886) +- feat(task): workspace support with --filter and --recursive (#26949) +- feat(watch): log which file changed on HMR or watch change (#25801) +- feat: OpenTelemetry Tracing API and Exporting (#26710) +- feat: Wasm module support (#26668) +- feat: fmt and lint respect .gitignore file (#26897) +- feat: permission stack traces in ops (#26938) +- feat: subcommand to view and update outdated dependencies (#26942) +- feat: upgrade V8 to 13.0 (#26851) +- fix(cli): preserve comments in doc tests (#26828) +- fix(cli): show prefix hint when installing a package globally (#26629) +- fix(ext/cache): gracefully error when cache creation failed (#26895) +- fix(ext/http): prefer brotli for `accept-encoding: gzip, deflate, br, zstd` + (#26814) +- fix(ext/node): New async setInterval function to improve the nodejs + compatibility (#26703) +- fix(ext/node): add autoSelectFamily option to net.createConnection (#26661) +- fix(ext/node): handle `--allow-sys=inspector` (#26836) +- fix(ext/node): increase tolerance for interval test (#26899) +- fix(ext/node): process.getBuiltinModule (#26833) +- fix(ext/node): use ERR_NOT_IMPLEMENTED for notImplemented (#26853) +- fix(ext/node): zlib.crc32() (#26856) +- fix(ext/webgpu): Create GPUQuerySet converter before usage (#26883) +- fix(ext/websocket): initialize `error` attribute of WebSocket ErrorEvent + (#26796) +- fix(ext/webstorage): use error class for sqlite error case (#26806) +- fix(fmt): error instead of panic on unstable format (#26859) +- fix(fmt): formatting of .svelte files (#26948) +- fix(install): percent encodings in interactive progress bar (#26600) +- fix(install): re-setup bin entries after running lifecycle scripts (#26752) +- fix(lockfile): track dependencies specified in TypeScript compiler options + (#26551) +- fix(lsp): ignore editor indent settings if deno.json is present (#26912) +- fix(lsp): skip code action edits that can't be converted (#26831) +- fix(node): handle resolving ".//" in npm packages (#26920) +- fix(node/crypto): support promisify on generateKeyPair (#26913) +- fix(permissions): say to use --allow-run instead of --allow-all (#26842) +- fix(publish): improve error message when missing exports (#26945) +- fix: otel resiliency (#26857) +- fix: update message for unsupported schemes with npm and jsr (#26884) +- perf(compile): code cache (#26528) +- perf(windows): delay load webgpu and some other dlls (#26917) +- perf: use available system memory for v8 isolate memory limit (#26868) + +### 2.0.6 / 2024.11.10 + +- feat(ext/http): abort event when request is cancelled (#26781) +- feat(ext/http): abort signal when request is cancelled (#26761) +- feat(lsp): auto-import completions from byonm dependencies (#26680) +- fix(ext/cache): don't panic when creating cache (#26780) +- fix(ext/node): better inspector support (#26471) +- fix(fmt): don't use self-closing tags in HTML (#26754) +- fix(install): cache jsr deps from all workspace config files (#26779) +- fix(node:zlib): gzip & gzipSync should accept ArrayBuffer (#26762) +- fix: performance.timeOrigin (#26787) + +### 2.0.5 / 2024.11.05 + +- fix(add): better error message when adding package that only has pre-release + versions (#26724) +- fix(add): only add npm deps to package.json if it's at least as close as + deno.json (#26683) +- fix(cli): set `npm_config_user_agent` when running npm packages or tasks + (#26639) +- fix(coverage): exclude comment lines from coverage reports (#25939) +- fix(ext/node): add `findSourceMap` to the default export of `node:module` + (#26720) +- fix(ext/node): convert errors from `fs.readFile/fs.readFileSync` to node + format (#26632) +- fix(ext/node): resolve exports even if parent module filename isn't present + (#26553) +- fix(ext/node): return `this` from `http.Server.ref/unref()` (#26647) +- fix(fmt): do not panic for jsx ignore container followed by jsx text (#26723) +- fix(fmt): fix several HTML and components issues (#26654) +- fix(fmt): ignore file directive for YAML files (#26717) +- fix(install): handle invalid function error, and fallback to junctions + regardless of the error (#26730) +- fix(lsp): include unstable features from editor settings (#26655) +- fix(lsp): scope attribution for lazily loaded assets (#26699) +- fix(node): Implement `os.userInfo` properly, add missing `toPrimitive` + (#24702) +- fix(serve): support serve hmr (#26078) +- fix(types): missing `import` permission on `PermissionOptionsObject` (#26627) +- fix(workspace): support wildcard packages (#26568) +- fix: clamp smi in fast calls by default (#26506) +- fix: improved support for cjs and cts modules (#26558) +- fix: op_run_microtasks crash (#26718) +- fix: panic_hook hangs without procfs (#26732) +- fix: remove permission check in op_require_node_module_paths (#26645) +- fix: surface package.json location on dep parse failure (#26665) +- perf(lsp): don't walk coverage directory (#26715) + +### 2.0.4 / 2024.10.29 + +- Revert "fix(ext/node): fix dns.lookup result ordering (#26264)" (#26621) +- Revert "fix(ext/node): use primordials in `ext/node/polyfills/https.ts` + (#26323)" (#26613) +- feat(lsp): "typescript.preferences.preferTypeOnlyAutoImports" setting (#26546) +- fix(check): expose more globals from @types/node (#26603) +- fix(check): ignore resolving `jsxImportSource` when jsx is not used in graph + (#26548) +- fix(cli): Make --watcher CLEAR_SCREEN clear scrollback buffer as well as + visible screen (#25997) +- fix(compile): regression handling redirects (#26586) +- fix(ext/napi): export dynamic symbols list for {Free,Open}BSD (#26605) +- fix(ext/node): add path to `fs.stat` and `fs.statSync` error (#26037) +- fix(ext/node): compatibility with {Free,Open}BSD (#26604) +- fix(ext/node): use primordials in + ext\node\polyfills\internal\crypto\_randomInt.ts (#26534) +- fix(install): cache json exports of JSR packages (#26552) +- fix(install): regression - do not panic when config file contains \r\n + newlines (#26547) +- fix(lsp): make missing import action fix infallible (#26539) +- fix(npm): match npm bearer token generation (#26544) +- fix(upgrade): stop running `deno lsp` processes on windows before attempting + to replace executable (#26542) +- fix(watch): don't panic on invalid file specifiers (#26577) +- fix: do not panic when failing to write to http cache (#26591) +- fix: provide hints in terminal errors for Node.js globals (#26610) +- fix: report exceptions from nextTick (#26579) +- fix: support watch flag to enable watching other files than the main module on + serve subcommand (#26622) +- perf: pass transpiled module to deno_core as known string (#26555) + +### 2.0.3 / 2024.10.25 + +- feat(lsp): interactive inlay hints (#26382) +- fix: support node-api in denort (#26389) +- fix(check): support `--frozen` on deno check (#26479) +- fix(cli): increase size of blocking task threadpool on windows (#26465) +- fix(config): schemas for lint rule and tag autocompletion (#26515) +- fix(ext/console): ignore casing for named colors in css parsing (#26466) +- fix(ext/ffi): return u64/i64 as bigints from nonblocking ffi calls (#26486) +- fix(ext/node): cancel pending ipc writes on channel close (#26504) +- fix(ext/node): map `ERROR_INVALID_NAME` to `ENOENT` on windows (#26475) +- fix(ext/node): only set our end of child process pipe to nonblocking mode + (#26495) +- fix(ext/node): properly map reparse point error in readlink (#26375) +- fix(ext/node): refactor http.ServerResponse into function class (#26210) +- fix(ext/node): stub HTTPParser internal binding (#26401) +- fix(ext/node): use primordials in `ext/node/polyfills/https.ts` (#26323) +- fix(fmt): --ext flag requires to pass files (#26525) +- fix(fmt): upgrade formatters (#26469) +- fix(help): missing package specifier (#26380) +- fix(info): resolve workspace member mappings (#26350) +- fix(install): better json editing (#26450) +- fix(install): cache all exports of JSR packages listed in `deno.json` (#26501) +- fix(install): cache type only module deps in `deno install` (#26497) +- fix(install): don't cache json exports of JSR packages (for now) (#26530) +- fix(install): update lockfile when using package.json (#26458) +- fix(lsp): import-map-remap quickfix for type imports (#26454) +- fix(node/util): support array formats in `styleText` (#26507) +- fix(node:tls): set TLSSocket.alpnProtocol for client connections (#26476) +- fix(npm): ensure scoped package name is encoded in URLs (#26390) +- fix(npm): support version ranges with && or comma (#26453) +- fix: `.npmrc` settings not being passed to install/add command (#26473) +- fix: add 'fmt-component' to unstable features in schema file (#26526) +- fix: share inotify fd across watchers (#26200) +- fix: unpin tokio version (#26457) +- perf(compile): pass module source data from binary directly to v8 (#26494) +- perf: avoid multiple calls to runMicrotask (#26378) + +### 2.0.2 / 2024.10.17 + +- fix(cli): set napi object property properly (#26344) +- fix(ext/node): add null check for kStreamBaseField (#26368) +- fix(install): don't attempt to cache specifiers that point to directories + (#26369) +- fix(jupyter): fix panics for overslow subtraction (#26371) +- fix(jupyter): update to the new logo (#26353) +- fix(net): don't try to set nodelay on upgrade streams (#26342) +- fix(node/fs): copyFile with `COPYFILE_EXCL` should not throw if the + destination doesn't exist (#26360) +- fix(node/http): normalize header names in `ServerResponse` (#26339) +- fix(runtime): send ws ping frames from inspector server (#26352) +- fix: don't warn on ignored signals on windows (#26332) + +### 2.0.1 / 2024.10.16 + +- feat(lsp): "deno/didRefreshDenoConfigurationTree" notifications (#26215) +- feat(unstable): `--unstable-detect-cjs` for respecting explicit + `"type": "commonjs"` (#26149) +- fix(add): create deno.json when running `deno add jsr:` (#26275) +- fix(add): exact version should not have range `^` specifier (#26302) +- fix(child_process): map node `--no-warnings` flag to `--quiet` (#26288) +- fix(cli): add prefix to install commands in help (#26318) +- fix(cli): consolidate pkg parser for install & remove (#26298) +- fix(cli): named export takes precedence over default export in doc testing + (#26112) +- fix(cli): improve deno info output for npm packages (#25906) +- fix(console/ext/repl): support using parseFloat() (#25900) +- fix(ext/console): apply coloring for console.table (#26280) +- fix(ext/napi): pass user context to napi_threadsafe_fn finalizers (#26229) +- fix(ext/node): allow writing to tty columns (#26201) +- fix(ext/node): compute pem length (upper bound) for key exports (#26231) +- fix(ext/node): fix dns.lookup result ordering (#26264) +- fix(ext/node): handle http2 server ending stream (#26235) +- fix(ext/node): implement TCP.setNoDelay (#26263) +- fix(ext/node): timingSafeEqual account for AB byteOffset (#26292) +- fix(ext/node): use primordials in `ext/node/polyfills/internal/buffer.mjs` + (#24993) +- fix(ext/webgpu): allow GL backend on Windows (#26206) +- fix(install): duplicate dependencies in `package.json` (#26128) +- fix(install): handle pkg with dep on self when pkg part of peer dep resolution + (#26277) +- fix(install): retry downloads of registry info / tarballs (#26278) +- fix(install): support installing npm package with alias (#26246) +- fix(jupyter): copy kernels icons to the kernel directory (#26084) +- fix(jupyter): keep running event loop when waiting for messages (#26049) +- fix(lsp): relative completions for bare import-mapped specifiers (#26137) +- fix(node): make `process.stdout.isTTY` writable (#26130) +- fix(node/util): export `styleText` from `node:util` (#26194) +- fix(npm): support `--allow-scripts` on `deno run` (and `deno add`, + `deno test`, etc) (#26075) +- fix(repl): importing json files (#26053) +- fix(repl): remove check flags (#26140) +- fix(unstable/worker): ensure import permissions are passed (#26101) +- fix: add hint for missing `document` global in terminal error (#26218) +- fix: do not panic on wsl share file paths on windows (#26081) +- fix: do not panic running remote cjs module (#26259) +- fix: do not panic when using methods on classes and interfaces in deno doc + html output (#26100) +- fix: improve suggestions and hints when using CommonJS modules (#26287) +- fix: node-api function call should use preamble (#26297) +- fix: panic in `prepare_stack_trace_callback` when global interceptor throws + (#26241) +- fix: use syntect for deno doc html generation (#26322) +- perf(http): avoid clone getting request method and url (#26250) +- perf(http): cache webidl.converters lookups in ext/fetch/23_response.js + (#26256) +- perf(http): make heap allocation for path conditional (#26289) +- perf: use fast calls for microtask ops (#26236) + +### 2.0.0 / 2024.10.09 + +Read announcement blog post at: https://deno.com/blog/v2 + +- BREAKING: `DENO_FUTURE=1` by default, or welcome to Deno 2.0 (#25213) +- BREAKING: disallow `new Deno.FsFile()` (#25478) +- BREAKING: drop support for Deno.run.{clearEnv,gid,uid} (#25371) +- BREAKING: improve types for `Deno.serve` (#25369) +- BREAKING: improved error code accuracy (#25383) +- BREAKING: make supported compilerOptions an allow list (#25432) +- BREAKING: move `width` and `height` options to `UnsafeWindowSurface` + constructor (#24200) +- BREAKING: remove --allow-hrtime (#25367) +- BREAKING: remove "emit" and "map" from deno info output (#25468) +- BREAKING: remove `--allow-none` flag (#25337) +- BREAKING: remove `--jobs` flag (#25336) +- BREAKING: remove `--trace-ops` (#25344) +- BREAKING: remove `--ts` flag (#25338) +- BREAKING: remove `--unstable` flag (#25522) +- BREAKING: remove `deno bundle` (#25339) +- BREAKING: remove `deno vendor` (#25343) +- BREAKING: remove `Deno.[Tls]Listener.prototype.rid` (#25556) +- BREAKING: remove `Deno.{Conn,TlsConn,TcpConn,UnixConn}.prototype.rid` (#25446) +- BREAKING: remove `Deno.{Reader,Writer}[Sync]` and `Deno.Closer` (#25524) +- BREAKING: remove `Deno.Buffer` (#25441) +- BREAKING: remove `Deno.close()` (#25347) +- BREAKING: remove `Deno.ConnectTlsOptions.{certChain,certFile,privateKey}` and + `Deno.ListenTlsOptions.certChain,certFile,keyFile}` (#25525) +- BREAKING: remove `Deno.copy()` (#25345) +- BREAKING: remove `Deno.customInspect` (#25348) +- BREAKING: remove `Deno.fdatasync[Sync]()` (#25520) +- BREAKING: remove `Deno.File` (#25447) +- BREAKING: remove `Deno.flock[Sync]()` (#25350) +- BREAKING: remove `Deno.FsFile.prototype.rid` (#25499) +- BREAKING: remove `Deno.fstat[Sync]()` (#25351) +- BREAKING: remove `Deno.FsWatcher.prototype.rid` (#25444) +- BREAKING: remove `Deno.fsync[Sync]()` (#25448) +- BREAKING: remove `Deno.ftruncate[Sync]()` (#25412) +- BREAKING: remove `Deno.funlock[Sync]()` (#25442) +- BREAKING: remove `Deno.futime[Sync]()` (#25252) +- BREAKING: remove `Deno.iter[Sync]()` (#25346) +- BREAKING: remove `Deno.read[Sync]()` (#25409) +- BREAKING: remove `Deno.readAll[Sync]()` (#25386) +- BREAKING: remove `Deno.seek[Sync]()` (#25449) +- BREAKING: remove `Deno.Seeker[Sync]` (#25551) +- BREAKING: remove `Deno.shutdown()` (#25253) +- BREAKING: remove `Deno.write[Sync]()` (#25408) +- BREAKING: remove `Deno.writeAll[Sync]()` (#25407) +- BREAKING: remove deprecated `UnsafeFnPointer` constructor type with untyped + `Deno.PointerObject` parameter (#25577) +- BREAKING: remove deprecated files config (#25535) +- BREAKING: Remove obsoleted Temporal APIs part 2 (#25505) +- BREAKING: remove remaining web types for compatibility (#25334) +- BREAKING: remove support for remote import maps in deno.json (#25836) +- BREAKING: rename "deps" remote cache folder to "remote" (#25969) +- BREAKING: soft-remove `Deno.isatty()` (#25410) +- BREAKING: soft-remove `Deno.run()` (#25403) +- BREAKING: soft-remove `Deno.serveHttp()` (#25451) +- BREAKING: undeprecate `Deno.FsWatcher.prototype.return()` (#25623) +- feat: add `--allow-import` flag (#25469) +- feat: Add a hint on error about 'Relative import path ... not prefixed with + ...' (#25430) +- feat: Add better error messages for unstable APIs (#25519) +- feat: Add suggestion for packages using Node-API addons (#25975) +- feat: Allow importing .cjs files (#25426) +- feat: default to TS for file extension and support ext flag in more scenarios + (#25472) +- feat: deprecate import assertions (#25281) +- feat: Don't warn about --allow-script when using esbuild (#25894) +- feat: hide several --unstable-* flags (#25378) +- feat: improve lockfile v4 to store normalized version constraints and be more + terse (#25247) +- feat: improve warnings for deprecations and lifecycle script for npm packages + (#25694) +- feat: include version number in all --json based outputs (#25335) +- feat: lockfile v4 by default (#25165) +- feat: make 'globalThis.location' a configurable property (#25812) +- feat: print `Listening on` messages on stderr instead of stdout (#25491) +- feat: remove `--lock-write` flag (#25214) +- feat: require jsr prefix for `deno install` and `deno add` (#25698) +- feat: require(esm) (#25501) +- feat: Show hints when using `window` global (#25805) +- feat: stabilize `Deno.createHttpClient()` (#25569) +- feat: suggest `deno install --entrypoint` instead of `deno cache` (#25228) +- feat: support DENO_LOG env var instead of RUST_LOG (#25356) +- feat: TypeScript 5.6 and `npm:@types/node@22` (#25614) +- feat: Update no-window lint rule (#25486) +- feat: update warning message for --allow-run with no list (#25693) +- feat: warn when using `--allow-run` with no allow list (#25215) +- feat(add): Add npm packages to package.json if present (#25477) +- feat(add): strip package subpath when adding a package (#25419) +- feat(add/install): Flag to add dev dependency to package.json (#25495) +- feat(byonm): support `deno run npm:` when package is not in + package.json (#25981) +- feat(check): turn on noImplicitOverride (#25695) +- feat(check): turn on useUnknownInCatchVariables (#25465) +- feat(cli): evaluate code snippets in JSDoc and markdown (#25220) +- feat(cli): give access to `process` global everywhere (#25291) +- feat(cli): use NotCapable error for permission errors (#25431) +- feat(config): Node modules option for 2.0 (#25299) +- feat(ext/crypto): import and export p521 keys (#25789) +- feat(ext/crypto): X448 support (#26043) +- feat(ext/kv): configurable limit params (#25174) +- feat(ext/node): add abort helpers, process & streams fix (#25262) +- feat(ext/node): add rootCertificates to node:tls (#25707) +- feat(ext/node): buffer.transcode() (#25972) +- feat(ext/node): export 'promises' symbol from 'node:timers' (#25589) +- feat(ext/node): export missing constants from 'zlib' module (#25584) +- feat(ext/node): export missing symbols from domain, puncode, repl, tls + (#25585) +- feat(ext/node): export more symbols from streams and timers/promises (#25582) +- feat(ext/node): expose ES modules for _ modules (#25588) +- feat(flags): allow double commas to escape values in path based flags (#25453) +- feat(flags): support user provided args in repl subcommand (#25605) +- feat(fmt): better error on malfored HTML files (#25853) +- feat(fmt): stabilize CSS, HTML and YAML formatters (#25753) +- feat(fmt): support vto and njk extensions (#25831) +- feat(fmt): upgrade markup_fmt (#25768) +- feat(install): deno install with entrypoint (#25411) +- feat(install): warn repeatedly about not-run lifecycle scripts on explicit + installs (#25878) +- feat(lint): add `no-process-global` lint rule (#25709) +- feat(lsp): add a message when someone runs 'deno lsp' manually (#26051) +- feat(lsp): auto-import types with 'import type' (#25662) +- feat(lsp): html/css/yaml file formatting (#25353) +- feat(lsp): quick fix for @deno-types="npm:@types/*" (#25954) +- feat(lsp): turn on useUnknownInCatchVariables (#25474) +- feat(lsp): unstable setting as list (#25552) +- feat(permissions): `Deno.mainModule` doesn't require permissions (#25667) +- feat(permissions): allow importing from cdn.jsdelivr.net by default (#26013) +- feat(serve): Support second parameter in deno serve (#25606) +- feat(tools/doc): display subitems in symbol overviews where applicable + (#25885) +- feat(uninstall): alias to 'deno remove' if -g flag missing (#25461) +- feat(upgrade): better error message on failure (#25503) +- feat(upgrade): print info links for Deno 2 RC releases (#25225) +- feat(upgrade): support LTS release channel (#25123) +- fix: add link to env var docs (#25557) +- fix: add suggestion how to fix importing CJS module (#21764) +- fix: add test ensuring als works across dynamic import (#25593) +- fix: better error for Deno.UnsafeWindowSurface, correct HttpClient name, + cleanup unused code (#25833) +- fix: cjs resolution cases (#25739) +- fix: consistent with deno_config and treat `"experimentalDecorators"` as + deprecated (#25735) +- fix: delete old Deno 1.x headers file when loading cache (#25283) +- fix: do not panic running invalid file specifier (#25530) +- fix: don't include extensionless files in file collection for lint & fmt by + default (#25721) +- fix: don't prompt when using `Deno.permissions.request` with `--no-prompt` + (#25811) +- fix: eagerly error for specifier with empty version constraint (#25944) +- fix: enable `Win32_Security` feature in `windows-sys` (#26007) +- fix: error on unsupported compiler options (#25714) +- fix: error out if a valid flag is passed before a subcommand (#25830) +- fix: fix jupyter display function type (#25326) +- fix: Float16Array type (#25506) +- fix: handle showing warnings while the progress bar is shown (#25187) +- fix: Hide 'deno cache' from help output (#25960) +- fix: invalid ipv6 hostname on `deno serve` (#25482) +- fix: linux canonicalization checks (#24641) +- fix: lock down allow-run permissions more (#25370) +- fix: make some warnings more standard (#25324) +- fix: no cmd prefix in help output go links (#25459) +- fix: only enable byonm if workspace root has pkg json (#25379) +- fix: panic when require(esm) (#25769) +- fix: precompile preserve SVG camelCase attributes (#25945) +- fix: reland async context (#25140) +- fix: remove --allow-run warning when using deno without args or subcommand + (#25684) +- fix: remove entrypoint hack for Deno 2.0 (#25332) +- fix: remove recently added deno.json node_modules aliasing (#25542) +- fix: remove the typo in the help message (#25962) +- fix: removed unstable-htttp from deno help (#25216) +- fix: replace `npm install` hint with `deno install` hint (#25244) +- fix: trim space around DENO_AUTH_TOKENS (#25147) +- fix: update deno_doc (#25290) +- fix: Update deno_npm to fix `deno install` with crossws (#25837) +- fix: update hint for `deno add ` (#25455) +- fix: update malva in deno to support astro css comments (#25553) +- fix: update nodeModulesDir config JSON schema (#25653) +- fix: update patchver to 0.2 (#25952) +- fix: update sui to 0.4 (#25942) +- fix: upgrade deno_ast 0.42 (#25313) +- fix: upgrade deno_core to 0.307.0 (#25287) +- fix(add/install): default to "latest" tag for npm packages in + `deno add npm:pkg` (#25858) +- fix(bench): Fix table column alignments and NO_COLOR=1 (#25190) +- fix(BREAKING): make dns record types have consistent naming (#25357) +- fix(byonm): resolve npm deps of jsr deps (#25399) +- fix(check): ignore noImplicitOverrides in remote modules (#25854) +- fix(check): move is cjs check from resolving to loading (#25597) +- fix(check): properly surface dependency errors in types file of js file + (#25860) +- fix(cli): `deno task` exit with status 0 (#25637) +- fix(cli): Default to auto with --node-modules-dir flag (#25772) +- fix(cli): handle edge cases around `export`s in doc tests and default export + (#25720) +- fix(cli): Map error kind to `PermissionDenied` when symlinking fails due to + permissions (#25398) +- fix(cli): Only set allow net flag for deno serve if not already allowed all + (#25743) +- fix(cli): Warn on not-run lifecycle scripts with global cache (#25786) +- fix(cli/tools): correct `deno init --serve` template behavior (#25318) +- fix(compile): support 'deno compile' in RC and LTS releases (#25875) +- fix(config): validate export names (#25436) +- fix(coverage): ignore urls from doc testing (#25736) +- fix(doc): surface graph errors as warnings (#25888) +- fix(dts): stabilize `fetch` declaration for use with `Deno.HttpClient` + (#25683) +- fix(ext/console): more precision in console.time (#25723) +- fix(ext/console): prevent duplicate error printing when the cause is assigned + (#25327) +- fix(ext/crypto): ensure EC public keys are exported uncompressed (#25766) +- fix(ext/crypto): fix identity test for x25519 derive bits (#26011) +- fix(ext/crypto): reject empty usages in SubtleCrypto#importKey (#25759) +- fix(ext/crypto): support md4 digest algorithm (#25656) +- fix(ext/crypto): throw DataError for invalid EC key import (#25181) +- fix(ext/fetch): fix lowercase http_proxy classified as https (#25686) +- fix(ext/fetch): percent decode userinfo when parsing proxies (#25229) +- fix(ext/http): do not set localhost to hostname unnecessarily (#24777) +- fix(ext/http): gracefully handle Response.error responses (#25712) +- fix(ext/node): add `FileHandle#writeFile` (#25555) +- fix(ext/node): add `vm.constants` (#25630) +- fix(ext/node): Add missing `node:path` exports (#25567) +- fix(ext/node): Add missing node:fs and node:constants exports (#25568) +- fix(ext/node): add stubs for `node:trace_events` (#25628) +- fix(ext/node): attach console stream properties (#25617) +- fix(ext/node): avoid showing `UNKNOWN` error from TCP handle (#25550) +- fix(ext/node): close upgraded socket when the underlying http connection is + closed (#25387) +- fix(ext/node): delay accept() call 2 ticks in net.Server#listen (#25481) +- fix(ext/node): don't throw error for unsupported signal binding on windows + (#25699) +- fix(ext/node): emit `online` event after worker thread is initialized (#25243) +- fix(ext/node): export `process.allowedNodeEnvironmentFlags` (#25629) +- fix(ext/node): export JWK public key (#25239) +- fix(ext/node): export request and response clases from `http2` module (#25592) +- fix(ext/node): fix `Cipheriv#update(string, undefined)` (#25571) +- fix(ext/node): fix Decipheriv when autoPadding disabled (#25598) +- fix(ext/node): fix process.stdin.pause() (#25864) +- fix(ext/node): Fix vm sandbox object panic (#24985) +- fix(ext/node): http2session ready state (#25143) +- fix(ext/node): Implement detached option in `child_process` (#25218) +- fix(ext/node): import EC JWK keys (#25266) +- fix(ext/node): import JWK octet key pairs (#25180) +- fix(ext/node): import RSA JWK keys (#25267) +- fix(ext/node): register `node:wasi` built-in (#25134) +- fix(ext/node): remove unimplemented promiseHook stubs (#25979) +- fix(ext/node): report freemem() on Linux in bytes (#25511) +- fix(ext/node): Rewrite `node:v8` serialize/deserialize (#25439) +- fix(ext/node): session close during stream setup (#25170) +- fix(ext/node): Stream should be instance of EventEmitter (#25527) +- fix(ext/node): stub `inspector/promises` (#25635) +- fix(ext/node): stub `process.cpuUsage()` (#25462) +- fix(ext/node): stub cpu_info() for OpenBSD (#25807) +- fix(ext/node): support x509 certificates in `createPublicKey` (#25731) +- fix(ext/node): throw when loading `cpu-features` module (#25257) +- fix(ext/node): update aead-gcm-stream to 0.3 (#25261) +- fix(ext/node): use primordials in `ext/node/polyfills/console.ts` (#25572) +- fix(ext/node): use primordials in ext/node/polyfills/wasi.ts (#25608) +- fix(ext/node): validate input lengths in `Cipheriv` and `Decipheriv` (#25570) +- fix(ext/web): don't ignore capture in EventTarget.removeEventListener (#25788) +- fix(ext/webgpu): allow to build on unsupported platforms (#25202) +- fix(ext/webgpu): sync category comment (#25580) +- fix(ext/webstorage): make `getOwnPropertyDescriptor` with symbol return + `undefined` (#13348) +- fix(flags): --allow-all should conflict with lower permissions (#25909) +- fix(flags): don't treat empty run command as task subcommand (#25708) +- fix(flags): move some content from docs.deno.com into help output (#25951) +- fix(flags): properly error out for urls (#25770) +- fix(flags): require global flag for permission flags in install subcommand + (#25391) +- fix(fmt): --check was broken for CSS, YAML and HTML (#25848) +- fix(fmt): fix incorrect quotes in components (#25249) +- fix(fmt): fix tabs in YAML (#25536) +- fix(fmt/markdown): fix regression with multi-line footnotes and inline math + (#25222) +- fix(info): error instead of panic for npm specifiers when using byonm (#25947) +- fix(info): move "version" field to top of json output (#25890) +- fix(inspector): Fix panic when re-entering runtime ops (#25537) +- fix(install): compare versions directly to decide whether to create a child + node_modules dir for a workspace member (#26001) +- fix(install): Make sure target node_modules exists when symlinking (#25494) +- fix(install): recommend using `deno install -g` when using a single http url + (#25388) +- fix(install): store tags associated with package in node_modules dir (#26000) +- fix(install): surface package.json dependency errors (#26023) +- fix(install): Use relative symlinks in deno install (#25164) +- fix(installl): make bin entries executable even if not put in + `node_modules/.bin` (#25873) +- fix(jupyter): allow unstable flags (#25483) +- fix(lint): correctly handle old jsx in linter (#25902) +- fix(lint): support linting jsr pkg without version field (#25230) +- fix(lockfile): use loose deserialization for version constraints (#25660) +- fix(lsp): encode url parts before parsing as uri (#25509) +- fix(lsp): exclude missing import quick fixes with bad resolutions (#26025) +- fix(lsp): panic on url_to_uri() (#25238) +- fix(lsp): properly resolve jsxImportSource for caching (#25688) +- fix(lsp): update diagnostics on npm install (#25352) +- fix(napi): Don't run microtasks in napi_resolve_deferred (#25246) +- fix(napi): Fix worker threads importing already-loaded NAPI addon (#25245) +- fix(no-slow-types): better `override` handling (#25989) +- fix(node): Don't error out if we fail to statically analyze CJS re-export + (#25748) +- fix(node): fix worker_threads issues blocking Angular support (#26024) +- fix(node): implement libuv APIs needed to support `npm:sqlite3` (#25893) +- fix(node): Include "node" condition during CJS re-export analysis (#25785) +- fix(node): Pass NPM_PROCESS_STATE to subprocesses via temp file instead of env + var (#25896) +- fix(node/byonm): do not accidentally resolve bare node built-ins (#25543) +- fix(node/cluster): improve stubs to make log4js work (#25146) +- fix(npm): better error handling for remote npm deps (#25670) +- fix(npm): root package has peer dependency on itself (#26022) +- fix(permissions): disallow any `LD_` or `DYLD_` prefixed env var without full + --allow-run permissions (#25271) +- fix(permissions): disallow launching subprocess with LD_PRELOAD env var + without full run permissions (#25221) +- fix(publish): ensure provenance is spec compliant (#25200) +- fix(regression): do not expose resolved path in Deno.Command permission denied + error (#25434) +- fix(runtime): don't error `child.output()` on consumed stream (#25657) +- fix(runtime): use more null proto objects again (#25040) +- fix(runtime/web_worker): populate `SnapshotOptions` for `WebWorker` when + instantiated without snapshot (#25280) +- fix(task): correct name for scoped npm package binaries (#25390) +- fix(task): support tasks with colons in name in `deno run` (#25233) +- fix(task): use current executable for deno even when not named deno (#26019) +- fix(types): simplify mtls related types (#25658) +- fix(upgrade): more informative information on invalid version (#25319) +- fix(windows): Deno.Command - align binary resolution with linux and mac + (#25429) +- fix(workspace): handle when config has members when specified via --config + (#25988) +- perf: fast path for cached dyn imports (#25636) +- perf: Use -O3 for sui in release builds (#26010) +- perf(cache): single cache file for remote modules (#24983) +- perf(cache): single cache file for typescript emit (#24994) +- perf(ext/fetch): improve decompression throughput by upgrading `tower_http` + (#25806) +- perf(ext/node): reduce some allocations in require (#25197) +- perf(ext/web): optimize performance.measure() (#25774) + ### 1.46.3 / 2024.09.04 - feat(upgrade): print info links for Deno 2 RC releases (#25225) diff --git a/bench_util/Cargo.toml b/bench_util/Cargo.toml index 5a04ab53557923..4d2284b6d92309 100644 --- a/bench_util/Cargo.toml +++ b/bench_util/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_bench_util" -version = "0.162.0" +version = "0.174.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/cli/Cargo.toml b/cli/Cargo.toml index ec2243a818b7a7..dd75f903e4b7bc 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno" -version = "2.0.0-rc.5" +version = "2.1.2" authors.workspace = true default-run = "deno" edition.workspace = true @@ -38,6 +38,11 @@ path = "./bench/lsp_bench_standalone.rs" [features] default = ["upgrade", "__vendored_zlib_ng"] +# A feature that enables heap profiling with dhat on Linux. +# 1. Compile with `cargo build --profile=release-with-debug --features=dhat-heap` +# 2. Run the executable. It will output a dhat-heap.json file. +# 3. Open the json file in https://nnethercote.github.io/dh_view/dh_view.html +dhat-heap = ["dhat"] # A feature that enables the upgrade subcommand and the background check for # available updates (of deno binary). This is typically disabled for (Linux) # distribution packages. @@ -64,27 +69,27 @@ winres.workspace = true [dependencies] deno_ast = { workspace = true, features = ["bundler", "cjs", "codegen", "proposal", "react", "sourcemap", "transforms", "typescript", "view", "visit"] } -deno_cache_dir = { workspace = true } -deno_config = { version = "=0.35.0", features = ["workspace", "sync"] } +deno_cache_dir.workspace = true +deno_config.workspace = true deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } -deno_doc = { version = "0.148.0", features = ["html", "syntect"] } -deno_graph = { version = "=0.82.2" } -deno_lint = { version = "=0.67.0", features = ["docs"] } +deno_doc = { version = "=0.161.1", features = ["rust", "comrak"] } +deno_graph = { version = "=0.86.2" } +deno_lint = { version = "=0.68.0", features = ["docs"] } deno_lockfile.workspace = true -deno_npm = "=0.25.2" +deno_npm.workspace = true deno_package_json.workspace = true +deno_path_util.workspace = true +deno_resolver.workspace = true deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_semver.workspace = true -deno_task_shell = "=0.17.0" +deno_task_shell = "=0.18.1" +deno_telemetry.workspace = true deno_terminal.workspace = true -eszip = "=0.78.0" -libsui = "0.3.1" -napi_sym.workspace = true +libsui = "0.5.0" node_resolver.workspace = true anstream = "0.6.14" async-trait.workspace = true -base32.workspace = true base64.workspace = true bincode = "=1.3.3" bytes.workspace = true @@ -93,16 +98,17 @@ chrono = { workspace = true, features = ["now"] } clap = { version = "=4.5.16", features = ["env", "string", "wrap_help", "error-context"] } clap_complete = "=4.5.24" clap_complete_fig = "=4.5.2" -color-print = "0.3.5" +color-print.workspace = true console_static_text.workspace = true -dashmap = "5.5.3" +dashmap.workspace = true data-encoding.workspace = true +dhat = { version = "0.3.3", optional = true } dissimilar = "=1.0.4" dotenvy = "0.15.7" -dprint-plugin-json = "=0.19.3" -dprint-plugin-jupyter = "=0.1.3" +dprint-plugin-json = "=0.19.4" +dprint-plugin-jupyter = "=0.1.5" dprint-plugin-markdown = "=0.17.8" -dprint-plugin-typescript = "=0.93.0" +dprint-plugin-typescript = "=0.93.2" env_logger = "=0.10.0" fancy-regex = "=0.10.0" faster-hex.workspace = true @@ -116,15 +122,15 @@ http-body-util.workspace = true hyper-util.workspace = true import_map = { version = "=0.20.1", features = ["ext"] } indexmap.workspace = true -jsonc-parser.workspace = true -jupyter_runtime = { package = "runtimelib", version = "=0.14.0" } +jsonc-parser = { workspace = true, features = ["cst", "serde"] } +jupyter_runtime = { package = "runtimelib", version = "=0.19.0", features = ["tokio-runtime"] } lazy-regex.workspace = true libc.workspace = true libz-sys.workspace = true log = { workspace = true, features = ["serde"] } lsp-types.workspace = true -malva = "=0.10.1" -markup_fmt = "=0.13.0" +malva = "=0.11.0" +markup_fmt = "=0.16.0" memmem.workspace = true monch.workspace = true notify.workspace = true @@ -146,6 +152,7 @@ serde_repr.workspace = true sha2.workspace = true shell-escape = "=0.1.5" spki = { version = "0.7", features = ["pem"] } +sqlformat = "=0.3.2" strsim = "0.11.1" tar.workspace = true tempfile.workspace = true @@ -161,7 +168,6 @@ typed-arena = "=2.0.2" uuid = { workspace = true, features = ["serde"] } walkdir = "=2.3.2" which.workspace = true -yoke.workspace = true zeromq.workspace = true zip = { version = "2.1.6", default-features = false, features = ["deflate-flate2"] } zstd.workspace = true diff --git a/cli/args/deno_json.rs b/cli/args/deno_json.rs index e9ab0189f506b8..3e6eb617a60885 100644 --- a/cli/args/deno_json.rs +++ b/cli/args/deno_json.rs @@ -22,6 +22,8 @@ impl<'a> deno_config::fs::DenoConfigFs for DenoConfigFsAdapter<'a> { self .0 .read_text_file_lossy_sync(path, None) + // todo(https://github.com/denoland/deno_config/pull/140): avoid clone + .map(|s| s.into_owned()) .map_err(|err| err.into_io_error()) } @@ -70,7 +72,41 @@ pub fn deno_json_deps( let values = imports_values(config.json.imports.as_ref()) .into_iter() .chain(scope_values(config.json.scopes.as_ref())); - values_to_set(values) + let mut set = values_to_set(values); + + if let Some(serde_json::Value::Object(compiler_options)) = + &config.json.compiler_options + { + // add jsxImportSource + if let Some(serde_json::Value::String(value)) = + compiler_options.get("jsxImportSource") + { + if let Some(dep_req) = value_to_dep_req(value) { + set.insert(dep_req); + } + } + // add jsxImportSourceTypes + if let Some(serde_json::Value::String(value)) = + compiler_options.get("jsxImportSourceTypes") + { + if let Some(dep_req) = value_to_dep_req(value) { + set.insert(dep_req); + } + } + // add the dependencies in the types array + if let Some(serde_json::Value::Array(types)) = compiler_options.get("types") + { + for value in types { + if let serde_json::Value::String(value) = value { + if let Some(dep_req) = value_to_dep_req(value) { + set.insert(dep_req); + } + } + } + } + } + + set } fn imports_values(value: Option<&serde_json::Value>) -> Vec<&String> { @@ -98,15 +134,23 @@ fn values_to_set<'a>( ) -> HashSet { let mut entries = HashSet::new(); for value in values { - if let Ok(req_ref) = JsrPackageReqReference::from_str(value) { - entries.insert(JsrDepPackageReq::jsr(req_ref.into_inner().req)); - } else if let Ok(req_ref) = NpmPackageReqReference::from_str(value) { - entries.insert(JsrDepPackageReq::npm(req_ref.into_inner().req)); + if let Some(dep_req) = value_to_dep_req(value) { + entries.insert(dep_req); } } entries } +fn value_to_dep_req(value: &str) -> Option { + if let Ok(req_ref) = JsrPackageReqReference::from_str(value) { + Some(JsrDepPackageReq::jsr(req_ref.into_inner().req)) + } else if let Ok(req_ref) = NpmPackageReqReference::from_str(value) { + Some(JsrDepPackageReq::npm(req_ref.into_inner().req)) + } else { + None + } +} + pub fn check_warn_tsconfig(ts_config: &TsConfigForEmit) { if let Some(ignored_options) = &ts_config.maybe_ignored_options { log::warn!("{}", ignored_options); diff --git a/cli/args/flags.rs b/cli/args/flags.rs index 10fa07bed2a5b2..5ea28bfec19377 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::borrow::Cow; use std::collections::HashSet; use std::env; use std::ffi::OsString; @@ -28,13 +29,14 @@ use deno_config::glob::PathOrPatternSet; use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; -use deno_core::normalize_path; use deno_core::resolve_url_or_path; use deno_core::url::Url; use deno_graph::GraphKind; -use deno_runtime::deno_permissions::parse_sys_kind; +use deno_path_util::normalize_path; +use deno_path_util::url_to_file_path; use deno_runtime::deno_permissions::PermissionsOptions; -use deno_runtime::fs_util::specifier_to_file_path; +use deno_runtime::deno_permissions::SysDescriptor; +use deno_telemetry::OtelConfig; use log::debug; use log::Level; use serde::Deserialize; @@ -44,6 +46,7 @@ use crate::args::resolve_no_prompt; use crate::util::fs::canonicalize_path; use super::flags_net; +use super::jsr_url; #[derive(Clone, Debug, Default, Eq, PartialEq)] pub enum ConfigFlag { @@ -207,6 +210,7 @@ pub struct FmtFlags { pub no_semicolons: Option, pub watch: Option, pub unstable_component: bool, + pub unstable_sql: bool, } impl FmtFlags { @@ -218,6 +222,8 @@ impl FmtFlags { #[derive(Clone, Debug, Eq, PartialEq)] pub struct InitFlags { + pub package: Option, + pub package_args: Vec, pub dir: Option, pub lib: bool, pub serve: bool, @@ -376,6 +382,9 @@ pub struct TaskFlags { pub cwd: Option, pub task: Option, pub is_run: bool, + pub recursive: bool, + pub filter: Option, + pub eval: bool, } #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] @@ -424,6 +433,7 @@ pub struct PublishFlags { pub allow_slow_types: bool, pub allow_dirty: bool, pub no_provenance: bool, + pub set_version: Option, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -459,6 +469,7 @@ pub enum DenoSubcommand { Serve(ServeFlags), Task(TaskFlags), Test(TestFlags), + Outdated(OutdatedFlags), Types, Upgrade(UpgradeFlags), Vendor, @@ -466,6 +477,19 @@ pub enum DenoSubcommand { Help(HelpFlags), } +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum OutdatedKind { + Update { latest: bool }, + PrintOutdated { compatible: bool }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OutdatedFlags { + pub filters: Vec, + pub recursive: bool, + pub kind: OutdatedKind, +} + impl DenoSubcommand { pub fn is_run(&self) -> bool { matches!(self, Self::Run(_)) @@ -546,6 +570,8 @@ pub struct LifecycleScriptsConfig { pub allowed: PackagesAllowedScripts, pub initial_cwd: PathBuf, pub root_dir: PathBuf, + /// Part of an explicit `deno install` + pub explicit_install: bool, } #[derive(Debug, Clone, Eq, PartialEq, Default)] @@ -571,11 +597,21 @@ fn parse_packages_allowed_scripts(s: &str) -> Result { pub struct UnstableConfig { // TODO(bartlomieju): remove in Deno 2.5 pub legacy_flag_enabled: bool, // --unstable - pub bare_node_builtins: bool, // --unstable-bare-node-builts + pub bare_node_builtins: bool, + pub detect_cjs: bool, pub sloppy_imports: bool, pub features: Vec, // --unstabe-kv --unstable-cron } +#[derive(Clone, Debug, Eq, PartialEq, Default)] +pub struct InternalFlags { + /// Used when the language server is configured with an + /// explicit cache option. + pub cache_path: Option, + /// Only reads to the lockfile instead of writing to it. + pub lockfile_skip_write: bool, +} + #[derive(Clone, Debug, Eq, PartialEq, Default)] pub struct Flags { /// Vector of CLI arguments - these are user script arguments, all Deno @@ -587,9 +623,6 @@ pub struct Flags { pub ca_stores: Option>, pub ca_data: Option, pub cache_blocklist: Vec, - /// This is not exposed as an option in the CLI, it is used internally when - /// the language server is configured with an explicit cache option. - pub cache_path: Option, pub cached_only: bool, pub type_check_mode: TypeCheckMode, pub config_flag: ConfigFlag, @@ -598,9 +631,11 @@ pub struct Flags { pub enable_op_summary_metrics: bool, pub enable_testing_features: bool, pub ext: Option, + /// Flags that aren't exposed in the CLI, but are used internally. + pub internal: InternalFlags, pub ignore: Vec, pub import_map_path: Option, - pub env_file: Option, + pub env_file: Option>, pub inspect_brk: Option, pub inspect_wait: Option, pub inspect: Option, @@ -639,6 +674,7 @@ pub struct PermissionFlags { pub allow_write: Option>, pub deny_write: Option>, pub no_prompt: bool, + pub allow_import: Option>, } impl PermissionFlags { @@ -658,9 +694,10 @@ impl PermissionFlags { || self.deny_sys.is_some() || self.allow_write.is_some() || self.deny_write.is_some() + || self.allow_import.is_some() } - pub fn to_options(&self) -> PermissionsOptions { + pub fn to_options(&self, cli_arg_urls: &[Cow]) -> PermissionsOptions { fn handle_allow( allow_all: bool, value: Option, @@ -673,6 +710,42 @@ impl PermissionFlags { } } + fn handle_imports( + cli_arg_urls: &[Cow], + imports: Option>, + ) -> Option> { + if imports.is_some() { + return imports; + } + + let builtin_allowed_import_hosts = [ + "jsr.io:443", + "deno.land:443", + "esm.sh:443", + "cdn.jsdelivr.net:443", + "raw.githubusercontent.com:443", + "gist.githubusercontent.com:443", + ]; + + let mut imports = + Vec::with_capacity(builtin_allowed_import_hosts.len() + 1); + imports + .extend(builtin_allowed_import_hosts.iter().map(|s| s.to_string())); + + // also add the JSR_URL env var + if let Some(jsr_host) = allow_import_host_from_url(jsr_url()) { + imports.push(jsr_host); + } + // include the cli arg urls + for url in cli_arg_urls { + if let Some(host) = allow_import_host_from_url(url) { + imports.push(host); + } + } + + Some(imports) + } + PermissionsOptions { allow_all: self.allow_all, allow_env: handle_allow(self.allow_all, self.allow_env.clone()), @@ -689,11 +762,33 @@ impl PermissionFlags { deny_sys: self.deny_sys.clone(), allow_write: handle_allow(self.allow_all, self.allow_write.clone()), deny_write: self.deny_write.clone(), + allow_import: handle_imports( + cli_arg_urls, + handle_allow(self.allow_all, self.allow_import.clone()), + ), prompt: !resolve_no_prompt(self), } } } +/// Gets the --allow-import host from the provided url +fn allow_import_host_from_url(url: &Url) -> Option { + let host = url.host()?; + if let Some(port) = url.port() { + Some(format!("{}:{}", host, port)) + } else { + use deno_core::url::Host::*; + match host { + Domain(domain) if domain == "jsr.io" && url.scheme() == "https" => None, + _ => match url.scheme() { + "https" => Some(format!("{}:443", host)), + "http" => Some(format!("{}:80", host)), + _ => None, + }, + } + } +} + fn join_paths(allowlist: &[String], d: &str) -> String { allowlist .iter() @@ -881,9 +976,38 @@ impl Flags { _ => {} } + match &self.permissions.allow_import { + Some(allowlist) if allowlist.is_empty() => { + args.push("--allow-import".to_string()); + } + Some(allowlist) => { + let s = format!("--allow-import={}", allowlist.join(",")); + args.push(s); + } + _ => {} + } + args } + pub fn otel_config(&self) -> Option { + if self + .unstable_config + .features + .contains(&String::from("otel")) + { + Some(OtelConfig { + runtime_name: Cow::Borrowed("deno"), + runtime_version: Cow::Borrowed(crate::version::DENO_VERSION_INFO.deno), + deterministic: std::env::var("DENO_UNSTABLE_OTEL_DETERMINISTIC") + .is_ok(), + ..Default::default() + }) + } else { + None + } + } + /// Extract the paths the config file should be discovered from. /// /// Returns `None` if the config file should not be auto-discovered. @@ -928,7 +1052,7 @@ impl Flags { if module_specifier.scheme() == "file" || module_specifier.scheme() == "npm" { - if let Ok(p) = specifier_to_file_path(&module_specifier) { + if let Ok(p) = url_to_file_path(&module_specifier) { Some(vec![p.parent().unwrap().to_path_buf()]) } else { Some(vec![current_dir.to_path_buf()]) @@ -991,6 +1115,7 @@ impl Flags { self.permissions.allow_write = None; self.permissions.allow_sys = None; self.permissions.allow_ffi = None; + self.permissions.allow_import = None; } pub fn resolve_watch_exclude_set( @@ -1054,25 +1179,26 @@ static ENV_VARIABLES_HELP: &str = cstr!( Docs: https://docs.deno.com/go/env-vars DENO_AUTH_TOKENS A semi-colon separated list of bearer tokens and hostnames - to use when fetching remote modules from private repositories - (e.g. "abcde12345@deno.land;54321edcba@github.com") - DENO_CERT Load certificate authorities from PEM encoded file - DENO_DIR Set the cache directory - DENO_INSTALL_ROOT Set deno install's output directory - (defaults to $HOME/.deno/bin) - DENO_NO_PACKAGE_JSON Disables auto-resolution of package.json - DENO_NO_UPDATE_CHECK Set to disable checking if a newer Deno version is available - DENO_TLS_CA_STORE Comma-separated list of order dependent certificate stores. - Possible values: "system", "mozilla". - (defaults to "mozilla") - HTTP_PROXY Proxy address for HTTP requests - (module downloads, fetch) - HTTPS_PROXY Proxy address for HTTPS requests - (module downloads, fetch) - NO_COLOR Set to disable color - NO_PROXY Comma-separated list of hosts which do not use a proxy - (module downloads, fetch) - NPM_CONFIG_REGISTRY URL to use for the npm registry."# + to use when fetching remote modules from private repositories + (e.g. "abcde12345@deno.land;54321edcba@github.com") + DENO_CERT Load certificate authorities from PEM encoded file + DENO_DIR Set the cache directory + DENO_INSTALL_ROOT Set deno install's output directory + (defaults to $HOME/.deno/bin) + DENO_NO_PACKAGE_JSON Disables auto-resolution of package.json + DENO_NO_UPDATE_CHECK Set to disable checking if a newer Deno version is available + DENO_TLS_CA_STORE Comma-separated list of order dependent certificate stores. + DENO_TRACE_PERMISSIONS Environmental variable to enable stack traces in permission prompts. + Possible values: "system", "mozilla". + (defaults to "mozilla") + HTTP_PROXY Proxy address for HTTP requests + (module downloads, fetch) + HTTPS_PROXY Proxy address for HTTPS requests + (module downloads, fetch) + NO_COLOR Set to disable color + NO_PROXY Comma-separated list of hosts which do not use a proxy + (module downloads, fetch) + NPM_CONFIG_REGISTRY URL to use for the npm registry."# ); static DENO_HELP: &str = cstr!( @@ -1093,21 +1219,21 @@ static DENO_HELP: &str = cstr!( Dependency management: add Add dependencies - deno add @std/assert | deno add npm:express - install Install script as an executable - uninstall Uninstall a script previously installed with deno install + deno add jsr:@std/assert | deno add npm:express + install Installs dependencies either in the local project or globally to a bin directory + uninstall Uninstalls a dependency or an executable script in the installation root's bin directory + outdated Find and update outdated dependencies remove Remove dependencies from the configuration file Tooling: bench Run benchmarks deno bench bench.ts - cache Cache the dependencies check Type-check the dependencies clean Remove the cache directory compile Compile the script into a self contained executable deno compile main.ts | deno compile --target=x86_64-unknown-linux-gnu coverage Print coverage reports - doc Genereate and show documentation for a module or built-ins + doc Generate and show documentation for a module or built-ins deno doc | deno doc --json | deno doc --html mod.ts fmt Format source files deno fmt | deno fmt main.ts @@ -1183,28 +1309,7 @@ pub fn flags_from_vec(args: Vec) -> clap::error::Result { .get_arguments() .any(|arg| arg.get_id().as_str() == "unstable") { - subcommand = subcommand - .mut_arg("unstable", |arg| { - let new_help = arg - .get_help() - .unwrap() - .to_string() - .split_once("\n") - .unwrap() - .0 - .to_string(); - arg.help_heading(UNSTABLE_HEADING).help(new_help) - }) - .mut_args(|arg| { - // long_help here is being used as a metadata, see unstable args definition - if arg.get_help_heading() == Some(UNSTABLE_HEADING) - && arg.get_long_help().is_some() - { - arg.hide(false) - } else { - arg - } - }); + subcommand = enable_unstable(subcommand); } help_parse(&mut flags, subcommand); @@ -1280,7 +1385,7 @@ pub fn flags_from_vec(args: Vec) -> clap::error::Result { } match subcommand.as_str() { - "add" => add_parse(&mut flags, &mut m), + "add" => add_parse(&mut flags, &mut m)?, "remove" => remove_parse(&mut flags, &mut m), "bench" => bench_parse(&mut flags, &mut m)?, "bundle" => bundle_parse(&mut flags, &mut m), @@ -1293,23 +1398,24 @@ pub fn flags_from_vec(args: Vec) -> clap::error::Result { "doc" => doc_parse(&mut flags, &mut m)?, "eval" => eval_parse(&mut flags, &mut m)?, "fmt" => fmt_parse(&mut flags, &mut m)?, - "init" => init_parse(&mut flags, &mut m), + "init" => init_parse(&mut flags, &mut m)?, "info" => info_parse(&mut flags, &mut m)?, "install" => install_parse(&mut flags, &mut m)?, "json_reference" => json_reference_parse(&mut flags, &mut m, app), "jupyter" => jupyter_parse(&mut flags, &mut m), "lint" => lint_parse(&mut flags, &mut m)?, "lsp" => lsp_parse(&mut flags, &mut m), + "outdated" => outdated_parse(&mut flags, &mut m)?, "repl" => repl_parse(&mut flags, &mut m)?, "run" => run_parse(&mut flags, &mut m, app, false)?, "serve" => serve_parse(&mut flags, &mut m, app)?, - "task" => task_parse(&mut flags, &mut m), + "task" => task_parse(&mut flags, &mut m, app)?, "test" => test_parse(&mut flags, &mut m)?, "types" => types_parse(&mut flags, &mut m), "uninstall" => uninstall_parse(&mut flags, &mut m), "upgrade" => upgrade_parse(&mut flags, &mut m), "vendor" => vendor_parse(&mut flags, &mut m), - "publish" => publish_parse(&mut flags, &mut m), + "publish" => publish_parse(&mut flags, &mut m)?, _ => unreachable!(), } } else { @@ -1339,6 +1445,31 @@ pub fn flags_from_vec(args: Vec) -> clap::error::Result { Ok(flags) } +fn enable_unstable(command: Command) -> Command { + command + .mut_arg("unstable", |arg| { + let new_help = arg + .get_help() + .unwrap() + .to_string() + .split_once("\n") + .unwrap() + .0 + .to_string(); + arg.help_heading(UNSTABLE_HEADING).help(new_help) + }) + .mut_args(|arg| { + // long_help here is being used as a metadata, see unstable args definition + if arg.get_help_heading() == Some(UNSTABLE_HEADING) + && arg.get_long_help().is_some() + { + arg.hide(false) + } else { + arg + } + }) +} + macro_rules! heading { ($($name:ident = $title:expr),+; $total:literal) => { $(const $name: &str = $title;)+ @@ -1441,7 +1572,7 @@ pub fn clap_root() -> Command { ); run_args(Command::new("deno"), true) - .args(unstable_args(UnstableArgsConfig::ResolutionAndRuntime)) + .with_unstable_args(UnstableArgsConfig::ResolutionAndRuntime) .next_line_help(false) .bin_name("deno") .styles( @@ -1517,6 +1648,7 @@ pub fn clap_root() -> Command { .subcommand(json_reference_subcommand()) .subcommand(jupyter_subcommand()) .subcommand(uninstall_subcommand()) + .subcommand(outdated_subcommand()) .subcommand(lsp_subcommand()) .subcommand(lint_subcommand()) .subcommand(publish_subcommand()) @@ -1543,7 +1675,7 @@ fn command( ) -> Command { Command::new(name) .about(about) - .args(unstable_args(unstable_args_config)) + .with_unstable_args(unstable_args_config) } fn help_subcommand(app: &Command) -> Command { @@ -1571,10 +1703,13 @@ fn add_subcommand() -> Command { "add", cstr!( "Add dependencies to your configuration file. - deno add @std/path + deno add jsr:@std/path -You can add multiple dependencies at once: - deno add @std/path @std/assert" +You can also add npm packages: + deno add npm:react + +Or multiple dependencies at once: + deno add jsr:@std/path jsr:@std/assert npm:chalk" ), UnstableArgsConfig::None, ) @@ -1588,6 +1723,7 @@ You can add multiple dependencies at once: .action(ArgAction::Append), ) .arg(add_dev_arg()) + .arg(allow_scripts_arg()) }) } @@ -1630,7 +1766,7 @@ If you specify a directory instead of a file, the path is expanded to all contai UnstableArgsConfig::ResolutionAndRuntime, ) .defer(|cmd| { - runtime_args(cmd, true, false) + runtime_args(cmd, true, false, true) .arg(check_arg(true)) .arg( Arg::new("json") @@ -1707,6 +1843,7 @@ Future runs of this module will trigger no downloads or compilation unless --rel ) .arg(frozen_lockfile_arg()) .arg(allow_scripts_arg()) + .arg(allow_import_arg()) }) } @@ -1766,6 +1903,8 @@ Unless --reload is specified, this command will not re-download already cached d .required_unless_present("help") .value_hint(ValueHint::FilePath), ) + .arg(frozen_lockfile_arg()) + .arg(allow_import_arg()) } ) } @@ -1775,11 +1914,15 @@ fn compile_subcommand() -> Command { "compile", cstr!("Compiles the given script into a self contained executable. - deno compile -A jsr:@std/http/file-server + deno compile --allow-read --allow-net jsr:@std/http/file-server deno compile --output file_server jsr:@std/http/file-server Any flags specified which affect runtime behavior will be applied to the resulting binary. +This allows distribution of a Deno application to systems that do not have Deno installed. +Under the hood, it bundles a slimmed down version of the Deno runtime along with your +JavaScript or TypeScript code. + Cross-compiling to different target architectures is supported using the --target flag. On the first invocation with deno will download the proper binary and cache it in $DENO_DIR. @@ -1788,16 +1931,16 @@ On the first invocation with deno will download the proper binary and cache it i UnstableArgsConfig::ResolutionAndRuntime, ) .defer(|cmd| { - runtime_args(cmd, true, false) + runtime_args(cmd, true, false, true) .arg(check_arg(true)) .arg( Arg::new("include") .long("include") .help( - cstr!("Includes an additional module in the compiled executable's module graph. + cstr!("Includes an additional module or file/directory in the compiled executable. Use this flag if a dynamically imported module or a web worker main module - fails to load in the executable. This flag can be passed multiple times, - to include multiple additional modules.", + fails to load in the executable or to embed a file or directory in the executable. + This flag can be passed multiple times, to include multiple additional modules.", )) .action(ArgAction::Append) .value_hint(ValueHint::FilePath) @@ -1825,6 +1968,7 @@ On the first invocation with deno will download the proper binary and cache it i ]) .help_heading(COMPILE_HEADING), ) + .arg(no_code_cache_arg()) .arg( Arg::new("no-terminal") .long("no-terminal") @@ -1994,6 +2138,7 @@ Show documentation for runtime built-ins: .arg(no_lock_arg()) .arg(no_npm_arg()) .arg(no_remote_arg()) + .arg(allow_import_arg()) .arg( Arg::new("json") .long("json") @@ -2108,7 +2253,7 @@ This command has implicit access to all permissions. UnstableArgsConfig::ResolutionAndRuntime, ) .defer(|cmd| { - runtime_args(cmd, false, true) + runtime_args(cmd, false, true, true) .arg(check_arg(false)) .arg(executable_ext_arg()) .arg( @@ -2145,6 +2290,9 @@ Supported file types which are behind corresponding unstable flags (see formatti Format stdin and write to stdout: cat file.ts | deno fmt - +Check if the files are formatted: + deno fmt --check + Ignore formatting code by preceding it with an ignore comment: // deno-fmt-ignore @@ -2172,9 +2320,9 @@ Ignore formatting a file by adding an ignore comment at the top of the file: .value_parser([ "ts", "tsx", "js", "jsx", "md", "json", "jsonc", "css", "scss", "sass", "less", "html", "svelte", "vue", "astro", "yml", "yaml", - "ipynb", + "ipynb", "sql" ]) - .help_heading(FMT_HEADING), + .help_heading(FMT_HEADING).requires("files"), ) .arg( Arg::new("ignore") @@ -2291,14 +2439,34 @@ Ignore formatting a file by adding an ignore comment at the top of the file: .help_heading(FMT_HEADING) .hide(true), ) + .arg( + Arg::new("unstable-sql") + .long("unstable-sql") + .help("Enable formatting SQL files.") + .value_parser(FalseyValueParser::new()) + .action(ArgAction::SetTrue) + .help_heading(FMT_HEADING), + ) }) } fn init_subcommand() -> Command { - command("init", "Initialize a new project", UnstableArgsConfig::None).defer( + command("init", "scaffolds a basic Deno project with a script, test, and configuration file", UnstableArgsConfig::None).defer( |cmd| { cmd - .arg(Arg::new("dir").value_hint(ValueHint::DirPath)) + .arg(Arg::new("args") + .num_args(0..) + .action(ArgAction::Append) + .value_name("DIRECTORY OR PACKAGE") + .trailing_var_arg(true) + ) + .arg( + Arg::new("npm") + .long("npm") + .help("Generate a npm create-* project") + .conflicts_with_all(["lib", "serve"]) + .action(ArgAction::SetTrue), + ) .arg( Arg::new("lib") .long("lib") @@ -2340,7 +2508,7 @@ The following information is shown: .arg( location_arg() .conflicts_with("file") - .help("Show files used for origin bound APIs like the Web Storage API when running a script with '--location='") + .help(cstr!("Show files used for origin bound APIs like the Web Storage API when running a script with --location=<>")) ) .arg(no_check_arg().hide(true)) // TODO(lucacasonato): remove for 2.0 .arg(no_config_arg()) @@ -2358,6 +2526,7 @@ The following information is shown: .help("UNSTABLE: Outputs the information in JSON format") .action(ArgAction::SetTrue), )) + .arg(allow_import_arg()) } fn install_subcommand() -> Command { @@ -2370,7 +2539,7 @@ in the package cache. If no dependency is specified, installs all dependencies l If the --entrypoint flag is passed, installs the dependencies of the specified entrypoint(s). deno install - deno install @std/bytes + deno install jsr:@std/bytes deno install npm:chalk deno install --entrypoint entry1.ts entry2.ts @@ -2381,7 +2550,7 @@ If the --global flag is set, installs a script as an executable in the deno install --global --allow-net --allow-read jsr:@std/http/file-server deno install -g https://examples.deno.land/color-logging.ts -To change the executable name, use -n/--name: +To change the executable name, use -n/--name: deno install -g --allow-net --allow-read -n serve jsr:@std/http/file-server The executable name is inferred by default: @@ -2403,7 +2572,7 @@ The installation root is determined, in order of precedence: These must be added to the path manually if required."), UnstableArgsConfig::ResolutionAndRuntime) .visible_alias("i") .defer(|cmd| { - permission_args(runtime_args(cmd, false, true), Some("global")) + permission_args(runtime_args(cmd, false, true, false), Some("global")) .arg(check_arg(true)) .arg(allow_scripts_arg()) .arg( @@ -2485,6 +2654,83 @@ fn jupyter_subcommand() -> Command { .conflicts_with("install")) } +fn outdated_subcommand() -> Command { + command( + "outdated", + cstr!("Find and update outdated dependencies. +By default, outdated dependencies are only displayed. + +Display outdated dependencies: + deno outdated + deno outdated --compatible + +Update dependencies: + deno outdated --update + deno outdated --update --latest + deno outdated --update + +Filters can be used to select which packages to act on. Filters can include wildcards (*) to match multiple packages. + deno outdated --update --latest \"@std/*\" + deno outdated --update --latest \"react*\" +Note that filters act on their aliases configured in deno.json / package.json, not the actual package names: + Given \"foobar\": \"npm:react@17.0.0\" in deno.json or package.json, the filter \"foobar\" would update npm:react to + the latest version. + deno outdated --update --latest foobar +Filters can be combined, and negative filters can be used to exclude results: + deno outdated --update --latest \"@std/*\" \"!@std/fmt*\" + +Specific version requirements to update to can be specified: + deno outdated --update @std/fmt@^1.0.2 +"), + UnstableArgsConfig::None, + ) + .defer(|cmd| { + cmd + .arg( + Arg::new("filters") + .num_args(0..) + .action(ArgAction::Append) + .help(concat!("Filters selecting which packages to act on. Can include wildcards (*) to match multiple packages. ", + "If a version requirement is specified, the matching packages will be updated to the given requirement."), + ) + ) + .arg(no_lock_arg()) + .arg(lock_arg()) + .arg( + Arg::new("latest") + .long("latest") + .action(ArgAction::SetTrue) + .help( + "Update to the latest version, regardless of semver constraints", + ) + .requires("update") + .conflicts_with("compatible"), + ) + .arg( + Arg::new("update") + .long("update") + .short('u') + .action(ArgAction::SetTrue) + .conflicts_with("compatible") + .help("Update dependency versions"), + ) + .arg( + Arg::new("compatible") + .long("compatible") + .action(ArgAction::SetTrue) + .help("Only output versions that satisfy semver requirements") + .conflicts_with("update"), + ) + .arg( + Arg::new("recursive") + .long("recursive") + .short('r') + .action(ArgAction::SetTrue) + .help("include all workspace members"), + ) + }) +} + fn uninstall_subcommand() -> Command { command( "uninstall", @@ -2663,9 +2909,19 @@ To ignore linting on an entire file, you can add an ignore comment at the top of } fn repl_subcommand() -> Command { - command("repl", "Read Eval Print Loop", UnstableArgsConfig::ResolutionAndRuntime) - .defer(|cmd| runtime_args(cmd, true, true) - .arg(check_arg(false)) + command("repl", cstr!( + "Starts a read-eval-print-loop, which lets you interactively build up program state in the global context. +It is especially useful for quick prototyping and checking snippets of code. + +TypeScript is supported, however it is not type-checked, only transpiled." + ), UnstableArgsConfig::ResolutionAndRuntime) + .defer(|cmd| { + let cmd = compile_args_without_check_args(cmd); + let cmd = inspect_args(cmd); + let cmd = permission_args(cmd, None); + let cmd = runtime_misc_args(cmd); + + cmd .arg( Arg::new("eval-file") .long("eval-file") @@ -2684,7 +2940,7 @@ fn repl_subcommand() -> Command { .after_help(cstr!("Environment variables: DENO_REPL_HISTORY Set REPL history file path. History file is disabled when the value is empty. [default: $DENO_DIR/deno_history.txt]")) - ) + }) .arg(env_file_arg()) .arg( Arg::new("args") @@ -2696,7 +2952,7 @@ fn repl_subcommand() -> Command { } fn run_args(command: Command, top_level: bool) -> Command { - runtime_args(command, true, true) + runtime_args(command, true, true, true) .arg(check_arg(false)) .arg(watch_arg(true)) .arg(hmr_arg(true)) @@ -2746,19 +3002,17 @@ fn serve_subcommand() -> Command { The serve command uses the default exports of the main module to determine which servers to start. -See https://docs.deno.com/runtime/manual/tools/serve for more detailed information. - Start a server defined in server.ts: deno serve server.ts Start a server defined in server.ts, watching for changes and running on port 5050: deno serve --watch --port 5050 server.ts -Read more: https://docs.deno.com/go/serve"), UnstableArgsConfig::ResolutionAndRuntime), true, true) +Read more: https://docs.deno.com/go/serve"), UnstableArgsConfig::ResolutionAndRuntime), true, true, true) .arg( Arg::new("port") .long("port") - .help("The TCP port to serve on, defaulting to 8000. Pass 0 to pick a random free port") + .help(cstr!("The TCP port to serve on. Pass 0 to pick a random free port [default: 8000]")) .value_parser(value_parser!(u16)), ) .arg( @@ -2793,7 +3047,10 @@ fn task_subcommand() -> Command { deno task build List all available tasks: - deno task" + deno task + +Evaluate a task from string + deno task --eval \"echo $(pwd)\"" ), UnstableArgsConfig::ResolutionAndRuntime, ) @@ -2809,6 +3066,28 @@ List all available tasks: .help("Specify the directory to run the task in") .value_hint(ValueHint::DirPath), ) + .arg( + Arg::new("recursive") + .long("recursive") + .short('r') + .help("Run the task in all projects in the workspace") + .action(ArgAction::SetTrue), + ) + .arg( + Arg::new("filter") + .long("filter") + .short('f') + .help("Filter members of the workspace by name, implies --recursive flag") + .value_parser(value_parser!(String)), + ) + .arg( + Arg::new("eval") + .long("eval") + .help( + "Evaluate the passed value as if it was a task in a configuration file", + ).action(ArgAction::SetTrue) + ) + .arg(node_modules_dir_arg()) }) } @@ -2827,7 +3106,7 @@ or **/__tests__/**: UnstableArgsConfig::ResolutionAndRuntime ) .defer(|cmd| - runtime_args(cmd, true, true) + runtime_args(cmd, true, true, true) .arg(check_arg(true)) .arg( Arg::new("ignore") @@ -2968,11 +3247,13 @@ fn parallel_arg(descr: &str) -> Arg { fn types_subcommand() -> Command { command( "types", - "Print runtime TypeScript declarations. + cstr!( + "Print runtime TypeScript declarations. deno types > lib.deno.d.ts -The declaration file could be saved and used for typing information.", +The declaration file could be saved and used for typing information." + ), UnstableArgsConfig::None, ) } @@ -3082,15 +3363,15 @@ See the Deno 1.x to 2.x Migration Guide for migration instructions: https://docs } fn publish_subcommand() -> Command { - command("publish", "Publish the current working directory's package or workspace", UnstableArgsConfig::ResolutionOnly) + command("publish", "Publish the current working directory's package or workspace to JSR", UnstableArgsConfig::ResolutionOnly) .defer(|cmd| { cmd - .arg( - Arg::new("token") - .long("token") - .help("The API token to use when publishing. If unset, interactive authentication is be used") - .help_heading(PUBLISH_HEADING) - ) + .arg( + Arg::new("token") + .long("token") + .help("The API token to use when publishing. If unset, interactive authentication is be used") + .help_heading(PUBLISH_HEADING) + ) .arg(config_arg()) .arg(no_config_arg()) .arg( @@ -3098,29 +3379,38 @@ fn publish_subcommand() -> Command { .long("dry-run") .help("Prepare the package for publishing performing all checks and validations without uploading") .action(ArgAction::SetTrue) - .help_heading(PUBLISH_HEADING), + .help_heading(PUBLISH_HEADING), ) .arg( Arg::new("allow-slow-types") .long("allow-slow-types") .help("Allow publishing with slow types") .action(ArgAction::SetTrue) - .help_heading(PUBLISH_HEADING), + .help_heading(PUBLISH_HEADING), ) .arg( Arg::new("allow-dirty") .long("allow-dirty") .help("Allow publishing if the repository has uncommitted changed") .action(ArgAction::SetTrue) - .help_heading(PUBLISH_HEADING), - ).arg( - Arg::new("no-provenance") - .long("no-provenance") - .help(cstr!("Disable provenance attestation. + .help_heading(PUBLISH_HEADING), + ) + .arg( + Arg::new("no-provenance") + .long("no-provenance") + .help(cstr!("Disable provenance attestation. Enabled by default on Github actions, publicly links the package to where it was built and published from.")) - .action(ArgAction::SetTrue) - .help_heading(PUBLISH_HEADING) - ) + .action(ArgAction::SetTrue) + .help_heading(PUBLISH_HEADING) + ) + .arg( + Arg::new("set-version") + .long("set-version") + .help("Set version for a package to be published. + This flag can be used while publishing individual packages and cannot be used in a workspace.") + .value_name("VERSION") + .help_heading(PUBLISH_HEADING) + ) .arg(check_arg(/* type checks by default */ true)) .arg(no_check_arg()) }) @@ -3151,47 +3441,46 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { .after_help(cstr!(r#"Permission options: Docs: https://docs.deno.com/go/permissions - -A, --allow-all Allow all permissions. - --no-prompt Always throw if required permission wasn't passed. - Can also be set via the DENO_NO_PROMPT environment variable. - -R, --allow-read[=<...] Allow file system read access. Optionally specify allowed paths. - --allow-read | --allow-read="/etc,/var/log.txt" - -W, --allow-write[=<...] Allow file system write access. Optionally specify allowed paths. - --allow-write | --allow-write="/etc,/var/log.txt" - -N, --allow-net[=<...] Allow network access. Optionally specify allowed IP addresses and host names, with ports as necessary. - --allow-net | --allow-net="localhost:8080,deno.land" - -E, --allow-env[=<...] Allow access to environment variables. Optionally specify accessible environment variables. - --allow-env | --allow-env="PORT,HOME,PATH" - -S, --allow-sys[=<...] Allow access to OS information. Optionally allow specific APIs by function name. - --allow-sys | --allow-sys="systemMemoryInfo,osRelease" - --allow-run[=<...] Allow running subprocesses. Optionally specify allowed runnable program names. - --allow-run | --allow-run="whoami,ps" - --allow-ffi[=<...] (Unstable) Allow loading dynamic libraries. Optionally specify allowed directories or files. - --allow-ffi | --allow-ffi="./libfoo.so" - --deny-read[=<...] Deny file system read access. Optionally specify denied paths. - --deny-read | --deny-read="/etc,/var/log.txt" - --deny-write[=<...] Deny file system write access. Optionally specify denied paths. - --deny-write | --deny-write="/etc,/var/log.txt" - --deny-net[=<...] Deny network access. Optionally specify defined IP addresses and host names, with ports as necessary. - --deny-net | --deny-net="localhost:8080,deno.land" - --deny-env[=<...] Deny access to environment variables. Optionally specify inacessible environment variables. - --deny-env | --deny-env="PORT,HOME,PATH" - -S, --deny-sys[=<...] Deny access to OS information. Optionally deny specific APIs by function name. - --deny-sys | --deny-sys="systemMemoryInfo,osRelease" - --deny-run[=<...] Deny running subprocesses. Optionally specify denied runnable program names. - --deny-run | --deny-run="whoami,ps" - --deny-ffi[=<...] (Unstable) Deny loading dynamic libraries. Optionally specify denied directories or files. - --deny-ffi | --deny-ffi="./libfoo.so" + -A, --allow-all Allow all permissions. + --no-prompt Always throw if required permission wasn't passed. + Can also be set via the DENO_NO_PROMPT environment variable. + -R, --allow-read[=<...] Allow file system read access. Optionally specify allowed paths. + --allow-read | --allow-read="/etc,/var/log.txt" + -W, --allow-write[=<...] Allow file system write access. Optionally specify allowed paths. + --allow-write | --allow-write="/etc,/var/log.txt" + -I, --allow-import[=<...] Allow importing from remote hosts. Optionally specify allowed IP addresses and host names, with ports as necessary. + Default value: deno.land:443,jsr.io:443,esm.sh:443,cdn.jsdelivr.net:443,raw.githubusercontent.com:443,user.githubusercontent.com:443 + --allow-import | --allow-import="example.com,github.com" + -N, --allow-net[=<...] Allow network access. Optionally specify allowed IP addresses and host names, with ports as necessary. + --allow-net | --allow-net="localhost:8080,deno.land" + -E, --allow-env[=<...] Allow access to environment variables. Optionally specify accessible environment variables. + --allow-env | --allow-env="PORT,HOME,PATH" + -S, --allow-sys[=<...] Allow access to OS information. Optionally allow specific APIs by function name. + --allow-sys | --allow-sys="systemMemoryInfo,osRelease" + --allow-run[=<...] Allow running subprocesses. Optionally specify allowed runnable program names. + --allow-run | --allow-run="whoami,ps" + --allow-ffi[=<...] (Unstable) Allow loading dynamic libraries. Optionally specify allowed directories or files. + --allow-ffi | --allow-ffi="./libfoo.so" + --deny-read[=<...] Deny file system read access. Optionally specify denied paths. + --deny-read | --deny-read="/etc,/var/log.txt" + --deny-write[=<...] Deny file system write access. Optionally specify denied paths. + --deny-write | --deny-write="/etc,/var/log.txt" + --deny-net[=<...] Deny network access. Optionally specify defined IP addresses and host names, with ports as necessary. + --deny-net | --deny-net="localhost:8080,deno.land" + --deny-env[=<...] Deny access to environment variables. Optionally specify inacessible environment variables. + --deny-env | --deny-env="PORT,HOME,PATH" + --deny-sys[=<...] Deny access to OS information. Optionally deny specific APIs by function name. + --deny-sys | --deny-sys="systemMemoryInfo,osRelease" + --deny-run[=<...] Deny running subprocesses. Optionally specify denied runnable program names. + --deny-run | --deny-run="whoami,ps" + --deny-ffi[=<...] (Unstable) Deny loading dynamic libraries. Optionally specify denied directories or files. + --deny-ffi | --deny-ffi="./libfoo.so" + DENO_TRACE_PERMISSIONS Environmental variable to enable stack traces in permission prompts. + DENO_TRACE_PERMISSIONS=1 deno run main.ts "#)) .arg( { - let mut arg = Arg::new("allow-all") - .short('A') - .long("allow-all") - .action(ArgAction::SetTrue) - .help("Allow all permissions") - .hide(true) - ; + let mut arg = allow_all_arg().hide(true); if let Some(requires) = requires { arg = arg.requires(requires) } @@ -3200,7 +3489,7 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { ) .arg( { - let mut arg = Arg::new("allow-read") + let mut arg = Arg::new("allow-read") .long("allow-read") .short('R') .num_args(0..) @@ -3218,7 +3507,7 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { ) .arg( { - let mut arg = Arg::new("deny-read") + let mut arg = Arg::new("deny-read") .long("deny-read") .num_args(0..) .action(ArgAction::Append) @@ -3235,7 +3524,7 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { ) .arg( { - let mut arg = Arg::new("allow-write") + let mut arg = Arg::new("allow-write") .long("allow-write") .short('W') .num_args(0..) @@ -3253,7 +3542,7 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { ) .arg( { - let mut arg = Arg::new("deny-write") + let mut arg = Arg::new("deny-write") .long("deny-write") .num_args(0..) .action(ArgAction::Append) @@ -3270,7 +3559,7 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { ) .arg( { - let mut arg = Arg::new("allow-net") + let mut arg = Arg::new("allow-net") .long("allow-net") .short('N') .num_args(0..) @@ -3279,8 +3568,7 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { .value_name("IP_OR_HOSTNAME") .help("Allow network access. Optionally specify allowed IP addresses and host names, with ports as necessary") .value_parser(flags_net::validator) - .hide(true) - ; + .hide(true); if let Some(requires) = requires { arg = arg.requires(requires) } @@ -3289,7 +3577,7 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { ) .arg( { - let mut arg = Arg::new("deny-net") + let mut arg = Arg::new("deny-net") .long("deny-net") .num_args(0..) .use_value_delimiter(true) @@ -3372,7 +3660,7 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { .require_equals(true) .value_name("API_NAME") .help("Allow access to OS information. Optionally allow specific APIs by function name") - .value_parser(|key: &str| parse_sys_kind(key).map(ToString::to_string)) + .value_parser(|key: &str| SysDescriptor::parse(key.to_string()).map(|s| s.into_string())) .hide(true) ; if let Some(requires) = requires { @@ -3383,14 +3671,14 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { ) .arg( { - let mut arg = Arg::new("deny-sys") + let mut arg = Arg::new("deny-sys") .long("deny-sys") .num_args(0..) .use_value_delimiter(true) .require_equals(true) .value_name("API_NAME") .help("Deny access to OS information. Optionally deny specific APIs by function name") - .value_parser(|key: &str| parse_sys_kind(key).map(ToString::to_string)) + .value_parser(|key: &str| SysDescriptor::parse(key.to_string()).map(|s| s.into_string())) .hide(true) ; if let Some(requires) = requires { @@ -3418,7 +3706,7 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { ) .arg( { - let mut arg = Arg::new("deny-run") + let mut arg = Arg::new("deny-run") .long("deny-run") .num_args(0..) .use_value_delimiter(true) @@ -3474,8 +3762,7 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { .long("allow-hrtime") .action(ArgAction::SetTrue) .help("REMOVED in Deno 2.0") - .hide(true) - ; + .hide(true); if let Some(requires) = requires { arg = arg.requires(requires) } @@ -3488,8 +3775,7 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { .long("deny-hrtime") .action(ArgAction::SetTrue) .help("REMOVED in Deno 2.0") - .hide(true) - ; + .hide(true); if let Some(requires) = requires { arg = arg.requires(requires) } @@ -3509,12 +3795,41 @@ fn permission_args(app: Command, requires: Option<&'static str>) -> Command { arg } ) + .arg( + { + let mut arg = allow_import_arg().hide(true); + if let Some(requires) = requires { + // allow this for install --global + if requires != "global" { + arg = arg.requires(requires) + } + } + arg + } + ) +} + +fn allow_all_arg() -> Arg { + Arg::new("allow-all") + .short('A') + .long("allow-all") + .conflicts_with("allow-read") + .conflicts_with("allow-write") + .conflicts_with("allow-net") + .conflicts_with("allow-env") + .conflicts_with("allow-run") + .conflicts_with("allow-sys") + .conflicts_with("allow-ffi") + .conflicts_with("allow-import") + .action(ArgAction::SetTrue) + .help("Allow all permissions") } fn runtime_args( app: Command, include_perms: bool, include_inspector: bool, + include_allow_scripts: bool, ) -> Command { let app = compile_args(app); let app = if include_perms { @@ -3527,6 +3842,15 @@ fn runtime_args( } else { app }; + let app = if include_allow_scripts { + app.arg(allow_scripts_arg()) + } else { + app + }; + runtime_misc_args(app) +} + +fn runtime_misc_args(app: Command) -> Command { app .arg(frozen_lockfile_arg()) .arg(cached_only_arg()) @@ -3537,6 +3861,20 @@ fn runtime_args( .arg(strace_ops_arg()) } +fn allow_import_arg() -> Arg { + Arg::new("allow-import") + .long("allow-import") + .short('I') + .num_args(0..) + .use_value_delimiter(true) + .require_equals(true) + .value_name("IP_OR_HOSTNAME") + .help(cstr!( + "Allow importing from remote hosts. Optionally specify allowed IP addresses and host names, with ports as necessary. Default value: deno.land:443,jsr.io:443,esm.sh:443,cdn.jsdelivr.net:443,raw.githubusercontent.com:443,user.githubusercontent.com:443" + )) + .value_parser(flags_net::validator) +} + fn inspect_args(app: Command) -> Command { app .arg( @@ -3599,12 +3937,14 @@ fn env_file_arg() -> Arg { .help(cstr!( "Load environment variables from local file Only the first environment variable with a given key is used. - Existing process environment variables are not overwritten." + Existing process environment variables are not overwritten, so if variables with the same names already exist in the environment, their values will be preserved. + Where multiple declarations for the same environment variable exist in your .env file, the first one encountered is applied. This is determined by the order of the files you pass as arguments." )) .value_hint(ValueHint::FilePath) .default_missing_value(".env") .require_equals(true) .num_args(0..=1) + .action(ArgAction::Append) } fn reload_arg() -> Arg { @@ -3683,7 +4023,9 @@ fn location_arg() -> Arg { url.set_password(None).unwrap(); Ok(url) }) - .help("Value of 'globalThis.location' used by some web APIs") + .help(cstr!( + "Value of globalThis.location used by some web APIs" + )) .value_hint(ValueHint::Url) } @@ -3992,23 +4334,29 @@ enum UnstableArgsConfig { ResolutionAndRuntime, } -struct UnstableArgsIter { - idx: usize, - cfg: UnstableArgsConfig, +trait CommandExt { + fn with_unstable_args(self, cfg: UnstableArgsConfig) -> Self; } -impl Iterator for UnstableArgsIter { - type Item = Arg; +impl CommandExt for Command { + fn with_unstable_args(self, cfg: UnstableArgsConfig) -> Self { + let mut next_display_order = { + let mut value = 1000; + move || { + value += 1; + value + } + }; - fn next(&mut self) -> Option { - let arg = if self.idx == 0 { + let mut cmd = self.arg( Arg::new("unstable") - .long("unstable") - .help(cstr!("Enable all unstable features and APIs. Instead of using this flag, consider enabling individual unstable features + .long("unstable") + .help(cstr!("Enable all unstable features and APIs. Instead of using this flag, consider enabling individual unstable features To view the list of individual unstable feature flags, run this command again with --help=unstable")) - .action(ArgAction::SetTrue) - .hide(matches!(self.cfg, UnstableArgsConfig::None)) - } else if self.idx == 1 { + .action(ArgAction::SetTrue) + .hide(matches!(cfg, UnstableArgsConfig::None)) + .display_order(next_display_order()) + ).arg( Arg::new("unstable-bare-node-builtins") .long("unstable-bare-node-builtins") .help("Enable unstable bare node builtins feature") @@ -4016,20 +4364,36 @@ impl Iterator for UnstableArgsIter { .value_parser(FalseyValueParser::new()) .action(ArgAction::SetTrue) .hide(true) - .long_help(match self.cfg { + .long_help(match cfg { UnstableArgsConfig::None => None, UnstableArgsConfig::ResolutionOnly | UnstableArgsConfig::ResolutionAndRuntime => Some("true"), }) .help_heading(UNSTABLE_HEADING) - } else if self.idx == 2 { + .display_order(next_display_order()), + ).arg( + Arg::new("unstable-detect-cjs") + .long("unstable-detect-cjs") + .help("Treats ambiguous .js, .jsx, .ts, .tsx files as CommonJS modules in more cases") + .value_parser(FalseyValueParser::new()) + .action(ArgAction::SetTrue) + .hide(true) + .long_help(match cfg { + UnstableArgsConfig::None => None, + UnstableArgsConfig::ResolutionOnly + | UnstableArgsConfig::ResolutionAndRuntime => Some("true"), + }) + .help_heading(UNSTABLE_HEADING) + .display_order(next_display_order()) + ).arg( Arg::new("unstable-byonm") .long("unstable-byonm") .value_parser(FalseyValueParser::new()) .action(ArgAction::SetTrue) .hide(true) .help_heading(UNSTABLE_HEADING) - } else if self.idx == 3 { + .display_order(next_display_order()), + ).arg( Arg::new("unstable-sloppy-imports") .long("unstable-sloppy-imports") .help("Enable unstable resolving of specifiers by extension probing, .js to .ts, and directory probing") @@ -4037,40 +4401,39 @@ impl Iterator for UnstableArgsIter { .value_parser(FalseyValueParser::new()) .action(ArgAction::SetTrue) .hide(true) - .long_help(match self.cfg { + .long_help(match cfg { UnstableArgsConfig::None => None, UnstableArgsConfig::ResolutionOnly | UnstableArgsConfig::ResolutionAndRuntime => Some("true") }) .help_heading(UNSTABLE_HEADING) - } else if self.idx > 3 { - let granular_flag = crate::UNSTABLE_GRANULAR_FLAGS.get(self.idx - 4)?; - Arg::new(format!("unstable-{}", granular_flag.name)) - .long(format!("unstable-{}", granular_flag.name)) - .help(granular_flag.help_text) - .action(ArgAction::SetTrue) - .hide(true) - .help_heading(UNSTABLE_HEADING) - // we don't render long help, so using it here as a sort of metadata - .long_help(if granular_flag.show_in_help { - match self.cfg { - UnstableArgsConfig::None | UnstableArgsConfig::ResolutionOnly => { - None + .display_order(next_display_order()) + ); + + for granular_flag in crate::UNSTABLE_GRANULAR_FLAGS.iter() { + cmd = cmd.arg( + Arg::new(format!("unstable-{}", granular_flag.name)) + .long(format!("unstable-{}", granular_flag.name)) + .help(granular_flag.help_text) + .action(ArgAction::SetTrue) + .hide(true) + .help_heading(UNSTABLE_HEADING) + // we don't render long help, so using it here as a sort of metadata + .long_help(if granular_flag.show_in_help { + match cfg { + UnstableArgsConfig::None | UnstableArgsConfig::ResolutionOnly => { + None + } + UnstableArgsConfig::ResolutionAndRuntime => Some("true"), } - UnstableArgsConfig::ResolutionAndRuntime => Some("true"), - } - } else { - None - }) - } else { - return None; - }; - self.idx += 1; - Some(arg.display_order(self.idx + 1000)) - } -} + } else { + None + }) + .display_order(next_display_order()), + ); + } -fn unstable_args(cfg: UnstableArgsConfig) -> impl IntoIterator { - UnstableArgsIter { idx: 0, cfg } + cmd + } } fn allow_scripts_arg_parse( @@ -4092,8 +4455,13 @@ fn allow_scripts_arg_parse( Ok(()) } -fn add_parse(flags: &mut Flags, matches: &mut ArgMatches) { +fn add_parse( + flags: &mut Flags, + matches: &mut ArgMatches, +) -> clap::error::Result<()> { + allow_scripts_arg_parse(flags, matches)?; flags.subcommand = DenoSubcommand::Add(add_parse_inner(matches, None)); + Ok(()) } fn add_parse_inner( @@ -4113,13 +4481,38 @@ fn remove_parse(flags: &mut Flags, matches: &mut ArgMatches) { }); } +fn outdated_parse( + flags: &mut Flags, + matches: &mut ArgMatches, +) -> clap::error::Result<()> { + let filters = match matches.remove_many::("filters") { + Some(f) => f.collect(), + None => vec![], + }; + let recursive = matches.get_flag("recursive"); + let update = matches.get_flag("update"); + let kind = if update { + let latest = matches.get_flag("latest"); + OutdatedKind::Update { latest } + } else { + let compatible = matches.get_flag("compatible"); + OutdatedKind::PrintOutdated { compatible } + }; + flags.subcommand = DenoSubcommand::Outdated(OutdatedFlags { + filters, + recursive, + kind, + }); + Ok(()) +} + fn bench_parse( flags: &mut Flags, matches: &mut ArgMatches, ) -> clap::error::Result<()> { flags.type_check_mode = TypeCheckMode::Local; - runtime_args_parse(flags, matches, true, false)?; + runtime_args_parse(flags, matches, true, false, true)?; ext_arg_parse(flags, matches); // NOTE: `deno bench` always uses `--no-prompt`, tests shouldn't ever do @@ -4174,6 +4567,7 @@ fn cache_parse( unstable_args_parse(flags, matches, UnstableArgsConfig::ResolutionOnly); frozen_lockfile_arg_parse(flags, matches); allow_scripts_arg_parse(flags, matches)?; + allow_import_parse(flags, matches); let files = matches.remove_many::("file").unwrap().collect(); flags.subcommand = DenoSubcommand::Cache(CacheFlags { files }); Ok(()) @@ -4186,6 +4580,7 @@ fn check_parse( flags.type_check_mode = TypeCheckMode::Local; compile_args_without_check_parse(flags, matches)?; unstable_args_parse(flags, matches, UnstableArgsConfig::ResolutionAndRuntime); + frozen_lockfile_arg_parse(flags, matches); let files = matches.remove_many::("file").unwrap().collect(); if matches.get_flag("all") || matches.get_flag("remote") { flags.type_check_mode = TypeCheckMode::All; @@ -4195,6 +4590,7 @@ fn check_parse( doc: matches.get_flag("doc"), doc_only: matches.get_flag("doc-only"), }); + allow_import_parse(flags, matches); Ok(()) } @@ -4207,7 +4603,7 @@ fn compile_parse( matches: &mut ArgMatches, ) -> clap::error::Result<()> { flags.type_check_mode = TypeCheckMode::Local; - runtime_args_parse(flags, matches, true, false)?; + runtime_args_parse(flags, matches, true, false, true)?; let mut script = matches.remove_many::("script_arg").unwrap(); let source_file = script.next().unwrap(); @@ -4222,6 +4618,8 @@ fn compile_parse( }; ext_arg_parse(flags, matches); + flags.code_cache_enabled = !matches.get_flag("no-code-cache"); + flags.subcommand = DenoSubcommand::Compile(CompileFlags { source_file, output, @@ -4320,6 +4718,7 @@ fn doc_parse( no_lock_arg_parse(flags, matches); no_npm_arg_parse(flags, matches); no_remote_arg_parse(flags, matches); + allow_import_parse(flags, matches); let source_files_val = matches.remove_many::("source_file"); let source_files = if let Some(val) = source_files_val { @@ -4381,7 +4780,7 @@ fn eval_parse( flags: &mut Flags, matches: &mut ArgMatches, ) -> clap::error::Result<()> { - runtime_args_parse(flags, matches, false, true)?; + runtime_args_parse(flags, matches, false, true, false)?; unstable_args_parse(flags, matches, UnstableArgsConfig::ResolutionAndRuntime); flags.allow_all(); @@ -4421,6 +4820,7 @@ fn fmt_parse( let prose_wrap = matches.remove_one::("prose-wrap"); let no_semicolons = matches.remove_one::("no-semicolons"); let unstable_component = matches.get_flag("unstable-component"); + let unstable_sql = matches.get_flag("unstable-sql"); flags.subcommand = DenoSubcommand::Fmt(FmtFlags { check: matches.get_flag("check"), @@ -4433,16 +4833,49 @@ fn fmt_parse( no_semicolons, watch: watch_arg_parse(matches)?, unstable_component, + unstable_sql, }); Ok(()) } -fn init_parse(flags: &mut Flags, matches: &mut ArgMatches) { +fn init_parse( + flags: &mut Flags, + matches: &mut ArgMatches, +) -> Result<(), clap::Error> { + let mut lib = matches.get_flag("lib"); + let mut serve = matches.get_flag("serve"); + let mut dir = None; + let mut package = None; + let mut package_args = vec![]; + + if let Some(mut args) = matches.remove_many::("args") { + let name = args.next().unwrap(); + let mut args = args.collect::>(); + + if matches.get_flag("npm") { + package = Some(name); + package_args = args; + } else { + dir = Some(name); + + if !args.is_empty() { + args.insert(0, "init".to_string()); + let inner_matches = init_subcommand().try_get_matches_from_mut(args)?; + lib = inner_matches.get_flag("lib"); + serve = inner_matches.get_flag("serve"); + } + } + } + flags.subcommand = DenoSubcommand::Init(InitFlags { - dir: matches.remove_one::("dir"), - lib: matches.get_flag("lib"), - serve: matches.get_flag("serve"), + package, + package_args, + dir, + lib, + serve, }); + + Ok(()) } fn info_parse( @@ -4460,6 +4893,7 @@ fn info_parse( lock_args_parse(flags, matches); no_remote_arg_parse(flags, matches); no_npm_arg_parse(flags, matches); + allow_import_parse(flags, matches); let json = matches.get_flag("json"); flags.subcommand = DenoSubcommand::Info(InfoFlags { file: matches.remove_one::("file"), @@ -4473,7 +4907,7 @@ fn install_parse( flags: &mut Flags, matches: &mut ArgMatches, ) -> clap::error::Result<()> { - runtime_args_parse(flags, matches, true, true)?; + runtime_args_parse(flags, matches, true, true, false)?; let global = matches.get_flag("global"); if global { @@ -4495,6 +4929,7 @@ fn install_parse( force, }), }); + return Ok(()); } @@ -4532,7 +4967,7 @@ fn json_reference_parse( app.build(); fn serialize_command( - command: &mut Command, + mut command: Command, top_level: bool, ) -> deno_core::serde_json::Value { let args = command @@ -4540,7 +4975,7 @@ fn json_reference_parse( .filter(|arg| { !arg.is_hide_set() && if top_level { - true + arg.is_global_set() } else { !arg.is_global_set() } @@ -4549,40 +4984,49 @@ fn json_reference_parse( let name = arg.get_id().as_str(); let short = arg.get_short(); let long = arg.get_long(); - let aliases = arg.get_visible_aliases(); let required = arg.is_required_set(); - let help = arg.get_help().map(|help| help.to_string()); + let help = arg.get_help().map(|help| help.ansi().to_string()); + let help_heading = arg + .get_help_heading() + .map(|help_heading| help_heading.to_string()); let usage = arg.to_string(); json!({ "name": name, "short": short, "long": long, - "aliases": aliases, "required": required, "help": help, + "help_heading": help_heading, "usage": usage, }) }) .collect::>(); let name = command.get_name().to_string(); - let about = command.get_about().map(|about| about.to_string()); - let visible_aliases = command - .get_visible_aliases() - .map(|s| s.to_string()) - .collect::>(); - let usage = command.render_usage().to_string(); + let about = command.get_about().map(|about| about.ansi().to_string()); + let usage = command.render_usage().ansi().to_string(); let subcommands = command - .get_subcommands_mut() - .map(|command| serialize_command(command, false)) + .get_subcommands() + .map(|command| { + serialize_command( + if command + .get_arguments() + .any(|arg| arg.get_id().as_str() == "unstable") + { + enable_unstable(command.clone()) + } else { + command.clone() + }, + false, + ) + }) .collect::>(); json!({ "name": name, "about": about, - "visible_aliases": visible_aliases, "args": args, "subcommands": subcommands, "usage": usage, @@ -4590,7 +5034,7 @@ fn json_reference_parse( } flags.subcommand = DenoSubcommand::JSONReference(JSONReferenceFlags { - json: serialize_command(&mut app, true), + json: serialize_command(app, true), }) } @@ -4689,8 +5133,18 @@ fn repl_parse( flags: &mut Flags, matches: &mut ArgMatches, ) -> clap::error::Result<()> { - runtime_args_parse(flags, matches, true, true)?; - unsafely_ignore_certificate_errors_parse(flags, matches); + unstable_args_parse(flags, matches, UnstableArgsConfig::ResolutionAndRuntime); + compile_args_without_check_parse(flags, matches)?; + cached_only_arg_parse(flags, matches); + frozen_lockfile_arg_parse(flags, matches); + permission_args_parse(flags, matches)?; + inspect_arg_parse(flags, matches); + location_arg_parse(flags, matches); + v8_flags_arg_parse(flags, matches); + seed_arg_parse(flags, matches); + enable_testing_features_arg_parse(flags, matches); + env_file_arg_parse(flags, matches); + strace_ops_parse(flags, matches); let eval_files = matches .remove_many::("eval-file") @@ -4722,7 +5176,7 @@ fn run_parse( mut app: Command, bare: bool, ) -> clap::error::Result<()> { - runtime_args_parse(flags, matches, true, true)?; + runtime_args_parse(flags, matches, true, true, true)?; ext_arg_parse(flags, matches); flags.code_cache_enabled = !matches.get_flag("no-code-cache"); @@ -4763,7 +5217,7 @@ fn serve_parse( let worker_count = parallel_arg_parse(matches).map(|v| v.get()); - runtime_args_parse(flags, matches, true, true)?; + runtime_args_parse(flags, matches, true, true, true)?; // If the user didn't pass --allow-net, add this port to the network // allowlist. If the host is 0.0.0.0, we add :{port} and allow the same network perms // as if it was passed to --allow-net directly. @@ -4811,18 +5265,29 @@ fn serve_parse( Ok(()) } -fn task_parse(flags: &mut Flags, matches: &mut ArgMatches) { +fn task_parse( + flags: &mut Flags, + matches: &mut ArgMatches, + mut app: Command, +) -> clap::error::Result<()> { flags.config_flag = matches .remove_one::("config") .map(ConfigFlag::Path) .unwrap_or(ConfigFlag::Discover); unstable_args_parse(flags, matches, UnstableArgsConfig::ResolutionAndRuntime); + node_modules_arg_parse(flags, matches); + + let filter = matches.remove_one::("filter"); + let recursive = matches.get_flag("recursive") || filter.is_some(); let mut task_flags = TaskFlags { cwd: matches.remove_one::("cwd"), task: None, is_run: false, + recursive, + filter, + eval: matches.get_flag("eval"), }; if let Some((task, mut matches)) = matches.remove_subcommand() { @@ -4835,9 +5300,15 @@ fn task_parse(flags: &mut Flags, matches: &mut ArgMatches) { .flatten() .filter_map(|arg| arg.into_string().ok()), ); + } else if task_flags.eval { + return Err(app.find_subcommand_mut("task").unwrap().error( + clap::error::ErrorKind::MissingRequiredArgument, + "[TASK] must be specified when using --eval", + )); } flags.subcommand = DenoSubcommand::Task(task_flags); + Ok(()) } fn parallel_arg_parse(matches: &mut ArgMatches) -> Option { @@ -4857,7 +5328,7 @@ fn test_parse( matches: &mut ArgMatches, ) -> clap::error::Result<()> { flags.type_check_mode = TypeCheckMode::Local; - runtime_args_parse(flags, matches, true, true)?; + runtime_args_parse(flags, matches, true, true, true)?; ext_arg_parse(flags, matches); // NOTE: `deno test` always uses `--no-prompt`, tests shouldn't ever do @@ -4983,7 +5454,10 @@ fn vendor_parse(flags: &mut Flags, _matches: &mut ArgMatches) { flags.subcommand = DenoSubcommand::Vendor } -fn publish_parse(flags: &mut Flags, matches: &mut ArgMatches) { +fn publish_parse( + flags: &mut Flags, + matches: &mut ArgMatches, +) -> clap::error::Result<()> { flags.type_check_mode = TypeCheckMode::Local; // local by default unstable_args_parse(flags, matches, UnstableArgsConfig::ResolutionOnly); no_check_arg_parse(flags, matches); @@ -4996,7 +5470,10 @@ fn publish_parse(flags: &mut Flags, matches: &mut ArgMatches) { allow_slow_types: matches.get_flag("allow-slow-types"), allow_dirty: matches.get_flag("allow-dirty"), no_provenance: matches.get_flag("no-provenance"), + set_version: matches.remove_one::("set-version"), }); + + Ok(()) } fn compile_args_parse( @@ -5175,13 +5652,22 @@ fn permission_args_parse( } if matches.get_flag("allow-hrtime") || matches.get_flag("deny-hrtime") { - log::warn!("⚠️ Warning: `allow-hrtime` and `deny-hrtime` have been removed in Deno 2, as high resolution time is now always allowed."); + // use eprintln instead of log::warn because logging hasn't been initialized yet + #[allow(clippy::print_stderr)] + { + eprintln!( + "{} `allow-hrtime` and `deny-hrtime` have been removed in Deno 2, as high resolution time is now always allowed", + deno_runtime::colors::yellow("Warning") + ); + } } if matches.get_flag("allow-all") { flags.allow_all(); } + allow_import_parse(flags, matches); + if matches.get_flag("no-prompt") { flags.permissions.no_prompt = true; } @@ -5189,6 +5675,13 @@ fn permission_args_parse( Ok(()) } +fn allow_import_parse(flags: &mut Flags, matches: &mut ArgMatches) { + if let Some(imports_wl) = matches.remove_many::("allow-import") { + let imports_allowlist = flags_net::parse(imports_wl.collect()).unwrap(); + flags.permissions.allow_import = Some(imports_allowlist); + } +} + fn unsafely_ignore_certificate_errors_parse( flags: &mut Flags, matches: &mut ArgMatches, @@ -5206,6 +5699,7 @@ fn runtime_args_parse( matches: &mut ArgMatches, include_perms: bool, include_inspector: bool, + include_allow_scripts: bool, ) -> clap::error::Result<()> { unstable_args_parse(flags, matches, UnstableArgsConfig::ResolutionAndRuntime); compile_args_parse(flags, matches)?; @@ -5217,6 +5711,9 @@ fn runtime_args_parse( if include_inspector { inspect_arg_parse(flags, matches); } + if include_allow_scripts { + allow_scripts_arg_parse(flags, matches)?; + } location_arg_parse(flags, matches); v8_flags_arg_parse(flags, matches); seed_arg_parse(flags, matches); @@ -5237,7 +5734,9 @@ fn import_map_arg_parse(flags: &mut Flags, matches: &mut ArgMatches) { } fn env_file_arg_parse(flags: &mut Flags, matches: &mut ArgMatches) { - flags.env_file = matches.remove_one::("env-file"); + flags.env_file = matches + .get_many::("env-file") + .map(|values| values.cloned().collect()); } fn reload_arg_parse( @@ -5488,6 +5987,7 @@ fn unstable_args_parse( flags.unstable_config.bare_node_builtins = matches.get_flag("unstable-bare-node-builtins"); + flags.unstable_config.detect_cjs = matches.get_flag("unstable-detect-cjs"); flags.unstable_config.sloppy_imports = matches.get_flag("unstable-sloppy-imports"); @@ -6215,7 +6715,7 @@ mod tests { #[test] fn short_permission_flags() { - let r = flags_from_vec(svec!["deno", "run", "-RNESW", "gist.ts"]); + let r = flags_from_vec(svec!["deno", "run", "-RNESWI", "gist.ts"]); assert_eq!( r.unwrap(), Flags { @@ -6226,6 +6726,7 @@ mod tests { allow_read: Some(vec![]), allow_write: Some(vec![]), allow_env: Some(vec![]), + allow_import: Some(vec![]), allow_net: Some(vec![]), allow_sys: Some(vec![]), ..Default::default() @@ -6307,6 +6808,7 @@ mod tests { prose_wrap: None, no_semicolons: None, unstable_component: false, + unstable_sql: false, watch: Default::default(), }), ..Flags::default() @@ -6330,6 +6832,7 @@ mod tests { prose_wrap: None, no_semicolons: None, unstable_component: false, + unstable_sql: false, watch: Default::default(), }), ..Flags::default() @@ -6353,6 +6856,7 @@ mod tests { prose_wrap: None, no_semicolons: None, unstable_component: false, + unstable_sql: false, watch: Default::default(), }), ..Flags::default() @@ -6376,6 +6880,7 @@ mod tests { prose_wrap: None, no_semicolons: None, unstable_component: false, + unstable_sql: false, watch: Some(Default::default()), }), ..Flags::default() @@ -6390,7 +6895,8 @@ mod tests { "--unstable-css", "--unstable-html", "--unstable-component", - "--unstable-yaml" + "--unstable-yaml", + "--unstable-sql" ]); assert_eq!( r.unwrap(), @@ -6408,6 +6914,7 @@ mod tests { prose_wrap: None, no_semicolons: None, unstable_component: true, + unstable_sql: true, watch: Some(WatchFlags { hmr: false, no_clear_screen: true, @@ -6442,6 +6949,7 @@ mod tests { prose_wrap: None, no_semicolons: None, unstable_component: false, + unstable_sql: false, watch: Some(Default::default()), }), ..Flags::default() @@ -6465,6 +6973,7 @@ mod tests { prose_wrap: None, no_semicolons: None, unstable_component: false, + unstable_sql: false, watch: Default::default(), }), config_flag: ConfigFlag::Path("deno.jsonc".to_string()), @@ -6496,6 +7005,7 @@ mod tests { prose_wrap: None, no_semicolons: None, unstable_component: false, + unstable_sql: false, watch: Some(Default::default()), }), config_flag: ConfigFlag::Path("deno.jsonc".to_string()), @@ -6532,6 +7042,7 @@ mod tests { prose_wrap: Some("never".to_string()), no_semicolons: Some(true), unstable_component: false, + unstable_sql: false, watch: Default::default(), }), ..Flags::default() @@ -6562,11 +7073,39 @@ mod tests { prose_wrap: None, no_semicolons: Some(false), unstable_component: false, + unstable_sql: false, watch: Default::default(), }), ..Flags::default() } ); + + let r = flags_from_vec(svec!["deno", "fmt", "--ext", "html"]); + assert!(r.is_err()); + let r = flags_from_vec(svec!["deno", "fmt", "--ext", "html", "./**"]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Fmt(FmtFlags { + check: false, + files: FileFlags { + include: vec!["./**".to_string()], + ignore: vec![], + }, + use_tabs: None, + line_width: None, + indent_width: None, + single_quote: None, + prose_wrap: None, + no_semicolons: None, + unstable_component: false, + unstable_sql: false, + watch: Default::default(), + }), + ext: Some("html".to_string()), + ..Flags::default() + } + ); } #[test] @@ -7146,7 +7685,7 @@ mod tests { allow_all: true, ..Default::default() }, - env_file: Some(".example.env".to_owned()), + env_file: Some(vec![".example.env".to_owned()]), ..Flags::default() } ); @@ -7215,7 +7754,7 @@ mod tests { #[test] fn repl_with_flags() { #[rustfmt::skip] - let r = flags_from_vec(svec!["deno", "repl", "-A", "--import-map", "import_map.json", "--no-remote", "--config", "tsconfig.json", "--no-check", "--reload", "--lock", "lock.json", "--cert", "example.crt", "--cached-only", "--location", "https:foo", "--v8-flags=--help", "--seed", "1", "--inspect=127.0.0.1:9229", "--unsafely-ignore-certificate-errors", "--env=.example.env"]); + let r = flags_from_vec(svec!["deno", "repl", "-A", "--import-map", "import_map.json", "--no-remote", "--config", "tsconfig.json", "--reload", "--lock", "lock.json", "--cert", "example.crt", "--cached-only", "--location", "https:foo", "--v8-flags=--help", "--seed", "1", "--inspect=127.0.0.1:9229", "--unsafely-ignore-certificate-errors", "--env=.example.env"]); assert_eq!( r.unwrap(), Flags { @@ -7240,7 +7779,7 @@ mod tests { allow_all: true, ..Default::default() }, - env_file: Some(".example.env".to_owned()), + env_file: Some(vec![".example.env".to_owned()]), unsafely_ignore_certificate_errors: Some(vec![]), ..Flags::default() } @@ -7263,7 +7802,6 @@ mod tests { allow_write: Some(vec![]), ..Default::default() }, - type_check_mode: TypeCheckMode::None, ..Flags::default() } ); @@ -7285,7 +7823,6 @@ mod tests { eval: None, is_default_command: false, }), - type_check_mode: TypeCheckMode::None, ..Flags::default() } ); @@ -7890,7 +8427,7 @@ mod tests { subcommand: DenoSubcommand::Run(RunFlags::new_default( "script.ts".to_string(), )), - env_file: Some(".env".to_owned()), + env_file: Some(vec![".env".to_owned()]), code_cache_enabled: true, ..Flags::default() } @@ -7906,7 +8443,7 @@ mod tests { subcommand: DenoSubcommand::Run(RunFlags::new_default( "script.ts".to_string(), )), - env_file: Some(".env".to_owned()), + env_file: Some(vec![".env".to_owned()]), code_cache_enabled: true, ..Flags::default() } @@ -7939,7 +8476,7 @@ mod tests { subcommand: DenoSubcommand::Run(RunFlags::new_default( "script.ts".to_string(), )), - env_file: Some(".another_env".to_owned()), + env_file: Some(vec![".another_env".to_owned()]), code_cache_enabled: true, ..Flags::default() } @@ -7960,7 +8497,29 @@ mod tests { subcommand: DenoSubcommand::Run(RunFlags::new_default( "script.ts".to_string(), )), - env_file: Some(".another_env".to_owned()), + env_file: Some(vec![".another_env".to_owned()]), + code_cache_enabled: true, + ..Flags::default() + } + ); + } + + #[test] + fn run_multiple_env_file_defined() { + let r = flags_from_vec(svec![ + "deno", + "run", + "--env-file", + "--env-file=.two_env", + "script.ts" + ]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Run(RunFlags::new_default( + "script.ts".to_string(), + )), + env_file: Some(vec![".env".to_owned(), ".two_env".to_owned()]), code_cache_enabled: true, ..Flags::default() } @@ -8103,7 +8662,7 @@ mod tests { allow_read: Some(vec![]), ..Default::default() }, - env_file: Some(".example.env".to_owned()), + env_file: Some(vec![".example.env".to_owned()]), ..Flags::default() } ); @@ -8687,8 +9246,12 @@ mod tests { #[test] fn test_no_colon_in_value_name() { - let app = - runtime_args(Command::new("test_inspect_completion_value"), true, true); + let app = runtime_args( + Command::new("test_inspect_completion_value"), + true, + true, + false, + ); let inspect_args = app .get_arguments() .filter(|arg| arg.get_id() == "inspect") @@ -9735,6 +10298,7 @@ mod tests { include: vec![] }), type_check_mode: TypeCheckMode::Local, + code_cache_enabled: true, ..Flags::default() } ); @@ -9743,7 +10307,7 @@ mod tests { #[test] fn compile_with_flags() { #[rustfmt::skip] - let r = flags_from_vec(svec!["deno", "compile", "--import-map", "import_map.json", "--no-remote", "--config", "tsconfig.json", "--no-check", "--unsafely-ignore-certificate-errors", "--reload", "--lock", "lock.json", "--cert", "example.crt", "--cached-only", "--location", "https:foo", "--allow-read", "--allow-net", "--v8-flags=--help", "--seed", "1", "--no-terminal", "--icon", "favicon.ico", "--output", "colors", "--env=.example.env", "https://examples.deno.land/color-logging.ts", "foo", "bar", "-p", "8080"]); + let r = flags_from_vec(svec!["deno", "compile", "--import-map", "import_map.json", "--no-code-cache", "--no-remote", "--config", "tsconfig.json", "--no-check", "--unsafely-ignore-certificate-errors", "--reload", "--lock", "lock.json", "--cert", "example.crt", "--cached-only", "--location", "https:foo", "--allow-read", "--allow-net", "--v8-flags=--help", "--seed", "1", "--no-terminal", "--icon", "favicon.ico", "--output", "colors", "--env=.example.env", "https://examples.deno.land/color-logging.ts", "foo", "bar", "-p", "8080"]); assert_eq!( r.unwrap(), Flags { @@ -9759,6 +10323,7 @@ mod tests { }), import_map_path: Some("import_map.json".to_string()), no_remote: true, + code_cache_enabled: false, config_flag: ConfigFlag::Path("tsconfig.json".to_owned()), type_check_mode: TypeCheckMode::None, reload: true, @@ -9774,7 +10339,7 @@ mod tests { unsafely_ignore_certificate_errors: Some(vec![]), v8_flags: svec!["--help", "--random-seed=1"], seed: Some(1), - env_file: Some(".example.env".to_owned()), + env_file: Some(vec![".example.env".to_owned()]), ..Flags::default() } ); @@ -9925,6 +10490,9 @@ mod tests { cwd: None, task: Some("build".to_string()), is_run: false, + recursive: false, + filter: None, + eval: false, }), argv: svec!["hello", "world"], ..Flags::default() @@ -9939,6 +10507,9 @@ mod tests { cwd: None, task: Some("build".to_string()), is_run: false, + recursive: false, + filter: None, + eval: false, }), ..Flags::default() } @@ -9952,10 +10523,80 @@ mod tests { cwd: Some("foo".to_string()), task: Some("build".to_string()), is_run: false, + recursive: false, + filter: None, + eval: false, + }), + ..Flags::default() + } + ); + + let r = flags_from_vec(svec!["deno", "task", "--filter", "*", "build"]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Task(TaskFlags { + cwd: None, + task: Some("build".to_string()), + is_run: false, + recursive: true, + filter: Some("*".to_string()), + eval: false, + }), + ..Flags::default() + } + ); + + let r = flags_from_vec(svec!["deno", "task", "--recursive", "build"]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Task(TaskFlags { + cwd: None, + task: Some("build".to_string()), + is_run: false, + recursive: true, + filter: None, + eval: false, + }), + ..Flags::default() + } + ); + + let r = flags_from_vec(svec!["deno", "task", "-r", "build"]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Task(TaskFlags { + cwd: None, + task: Some("build".to_string()), + is_run: false, + recursive: true, + filter: None, + eval: false, + }), + ..Flags::default() + } + ); + + let r = flags_from_vec(svec!["deno", "task", "--eval", "echo 1"]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Task(TaskFlags { + cwd: None, + task: Some("echo 1".to_string()), + is_run: false, + recursive: false, + filter: None, + eval: true, }), ..Flags::default() } ); + + let r = flags_from_vec(svec!["deno", "task", "--eval"]); + assert!(r.is_err()); } #[test] @@ -9977,6 +10618,9 @@ mod tests { cwd: None, task: Some("build".to_string()), is_run: false, + recursive: false, + filter: None, + eval: false, }), argv: svec!["--", "hello", "world"], config_flag: ConfigFlag::Path("deno.json".to_owned()), @@ -9994,6 +10638,9 @@ mod tests { cwd: Some("foo".to_string()), task: Some("build".to_string()), is_run: false, + recursive: false, + filter: None, + eval: false, }), argv: svec!["--", "hello", "world"], ..Flags::default() @@ -10012,6 +10659,9 @@ mod tests { cwd: None, task: Some("build".to_string()), is_run: false, + recursive: false, + filter: None, + eval: false, }), argv: svec!["--"], ..Flags::default() @@ -10029,6 +10679,9 @@ mod tests { cwd: None, task: Some("build".to_string()), is_run: false, + recursive: false, + filter: None, + eval: false, }), argv: svec!["-1", "--test"], ..Flags::default() @@ -10046,6 +10699,9 @@ mod tests { cwd: None, task: Some("build".to_string()), is_run: false, + recursive: false, + filter: None, + eval: false, }), argv: svec!["--test"], ..Flags::default() @@ -10064,6 +10720,9 @@ mod tests { cwd: None, task: Some("build".to_string()), is_run: false, + recursive: false, + filter: None, + eval: false, }), log_level: Some(log::Level::Error), ..Flags::default() @@ -10081,6 +10740,9 @@ mod tests { cwd: None, task: None, is_run: false, + recursive: false, + filter: None, + eval: false, }), ..Flags::default() } @@ -10097,6 +10759,9 @@ mod tests { cwd: None, task: None, is_run: false, + recursive: false, + filter: None, + eval: false, }), config_flag: ConfigFlag::Path("deno.jsonc".to_string()), ..Flags::default() @@ -10114,6 +10779,9 @@ mod tests { cwd: None, task: None, is_run: false, + recursive: false, + filter: None, + eval: false, }), config_flag: ConfigFlag::Path("deno.jsonc".to_string()), ..Flags::default() @@ -10290,6 +10958,8 @@ mod tests { r.unwrap(), Flags { subcommand: DenoSubcommand::Init(InitFlags { + package: None, + package_args: vec![], dir: None, lib: false, serve: false, @@ -10303,6 +10973,8 @@ mod tests { r.unwrap(), Flags { subcommand: DenoSubcommand::Init(InitFlags { + package: None, + package_args: vec![], dir: Some(String::from("foo")), lib: false, serve: false, @@ -10316,6 +10988,8 @@ mod tests { r.unwrap(), Flags { subcommand: DenoSubcommand::Init(InitFlags { + package: None, + package_args: vec![], dir: None, lib: false, serve: false, @@ -10330,6 +11004,8 @@ mod tests { r.unwrap(), Flags { subcommand: DenoSubcommand::Init(InitFlags { + package: None, + package_args: vec![], dir: None, lib: true, serve: false, @@ -10343,6 +11019,8 @@ mod tests { r.unwrap(), Flags { subcommand: DenoSubcommand::Init(InitFlags { + package: None, + package_args: vec![], dir: None, lib: false, serve: true, @@ -10356,6 +11034,8 @@ mod tests { r.unwrap(), Flags { subcommand: DenoSubcommand::Init(InitFlags { + package: None, + package_args: vec![], dir: Some(String::from("foo")), lib: true, serve: false, @@ -10363,6 +11043,57 @@ mod tests { ..Flags::default() } ); + + let r = flags_from_vec(svec!["deno", "init", "--lib", "--npm", "vite"]); + assert!(r.is_err()); + + let r = flags_from_vec(svec!["deno", "init", "--serve", "--npm", "vite"]); + assert!(r.is_err()); + + let r = flags_from_vec(svec!["deno", "init", "--npm", "vite", "--lib"]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Init(InitFlags { + package: Some("vite".to_string()), + package_args: svec!["--lib"], + dir: None, + lib: false, + serve: false, + }), + ..Flags::default() + } + ); + + let r = flags_from_vec(svec!["deno", "init", "--npm", "vite", "--serve"]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Init(InitFlags { + package: Some("vite".to_string()), + package_args: svec!["--serve"], + dir: None, + lib: false, + serve: false, + }), + ..Flags::default() + } + ); + + let r = flags_from_vec(svec!["deno", "init", "--npm", "vite", "new_dir"]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Init(InitFlags { + package: Some("vite".to_string()), + package_args: svec!["new_dir"], + dir: None, + lib: false, + serve: false, + }), + ..Flags::default() + } + ); } #[test] @@ -10436,6 +11167,7 @@ mod tests { "--allow-slow-types", "--allow-dirty", "--token=asdf", + "--set-version=1.0.1", ]); assert_eq!( r.unwrap(), @@ -10446,6 +11178,7 @@ mod tests { allow_slow_types: true, allow_dirty: true, no_provenance: true, + set_version: Some("1.0.1".to_string()), }), type_check_mode: TypeCheckMode::Local, ..Flags::default() @@ -10777,7 +11510,7 @@ mod tests { } ); // just make sure this doesn't panic - let _ = flags.permissions.to_options(); + let _ = flags.permissions.to_options(&[]); } #[test] @@ -10852,4 +11585,119 @@ mod tests { Usage: deno repl [OPTIONS] [-- [ARGS]...]\n" ) } + + #[test] + fn test_allow_import_host_from_url() { + fn parse(text: &str) -> Option { + allow_import_host_from_url(&Url::parse(text).unwrap()) + } + + assert_eq!(parse("https://jsr.io"), None); + assert_eq!( + parse("http://127.0.0.1:4250"), + Some("127.0.0.1:4250".to_string()) + ); + assert_eq!(parse("http://jsr.io"), Some("jsr.io:80".to_string())); + assert_eq!( + parse("https://example.com"), + Some("example.com:443".to_string()) + ); + assert_eq!( + parse("http://example.com"), + Some("example.com:80".to_string()) + ); + assert_eq!(parse("file:///example.com"), None); + } + + #[test] + fn allow_all_conflicts_allow_perms() { + let flags = [ + "--allow-read", + "--allow-write", + "--allow-net", + "--allow-env", + "--allow-run", + "--allow-sys", + "--allow-ffi", + "--allow-import", + ]; + for flag in flags { + let r = + flags_from_vec(svec!["deno", "run", "--allow-all", flag, "foo.ts"]); + assert!(r.is_err()); + } + } + + #[test] + fn outdated_subcommand() { + let cases = [ + ( + svec![], + OutdatedFlags { + filters: vec![], + kind: OutdatedKind::PrintOutdated { compatible: false }, + recursive: false, + }, + ), + ( + svec!["--recursive"], + OutdatedFlags { + filters: vec![], + kind: OutdatedKind::PrintOutdated { compatible: false }, + recursive: true, + }, + ), + ( + svec!["--recursive", "--compatible"], + OutdatedFlags { + filters: vec![], + kind: OutdatedKind::PrintOutdated { compatible: true }, + recursive: true, + }, + ), + ( + svec!["--update"], + OutdatedFlags { + filters: vec![], + kind: OutdatedKind::Update { latest: false }, + recursive: false, + }, + ), + ( + svec!["--update", "--latest"], + OutdatedFlags { + filters: vec![], + kind: OutdatedKind::Update { latest: true }, + recursive: false, + }, + ), + ( + svec!["--update", "--recursive"], + OutdatedFlags { + filters: vec![], + kind: OutdatedKind::Update { latest: false }, + recursive: true, + }, + ), + ( + svec!["--update", "@foo/bar"], + OutdatedFlags { + filters: svec!["@foo/bar"], + kind: OutdatedKind::Update { latest: false }, + recursive: false, + }, + ), + ]; + for (input, expected) in cases { + let mut args = svec!["deno", "outdated"]; + args.extend(input); + let r = flags_from_vec(args.clone()).unwrap(); + assert_eq!( + r.subcommand, + DenoSubcommand::Outdated(expected), + "incorrect result for args: {:?}", + args + ); + } + } } diff --git a/cli/args/flags_net.rs b/cli/args/flags_net.rs index 88ffcf0e468982..abfcf2838279ca 100644 --- a/cli/args/flags_net.rs +++ b/cli/args/flags_net.rs @@ -51,7 +51,7 @@ pub fn parse(paths: Vec) -> clap::error::Result> { } } else { NetDescriptor::parse(&host_and_port).map_err(|e| { - clap::Error::raw(clap::error::ErrorKind::InvalidValue, format!("{e:?}")) + clap::Error::raw(clap::error::ErrorKind::InvalidValue, e.to_string()) })?; out.push(host_and_port) } diff --git a/cli/args/lockfile.rs b/cli/args/lockfile.rs index 59ec7f0ef56060..a9eb8a0d7cae83 100644 --- a/cli/args/lockfile.rs +++ b/cli/args/lockfile.rs @@ -24,11 +24,20 @@ use crate::args::InstallKind; use deno_lockfile::Lockfile; +#[derive(Debug)] +pub struct CliLockfileReadFromPathOptions { + pub file_path: PathBuf, + pub frozen: bool, + /// Causes the lockfile to only be read from, but not written to. + pub skip_write: bool, +} + #[derive(Debug)] pub struct CliLockfile { lockfile: Mutex, pub filename: PathBuf, - pub frozen: bool, + frozen: bool, + skip_write: bool, } pub struct Guard<'a, T> { @@ -50,15 +59,6 @@ impl<'a, T> std::ops::DerefMut for Guard<'a, T> { } impl CliLockfile { - pub fn new(lockfile: Lockfile, frozen: bool) -> Self { - let filename = lockfile.filename.clone(); - Self { - lockfile: Mutex::new(lockfile), - filename, - frozen, - } - } - /// Get the inner deno_lockfile::Lockfile. pub fn lock(&self) -> Guard { Guard { @@ -78,6 +78,10 @@ impl CliLockfile { } pub fn write_if_changed(&self) -> Result<(), AnyError> { + if self.skip_write { + return Ok(()); + } + self.error_if_changed()?; let mut lockfile = self.lockfile.lock(); let Some(bytes) = lockfile.resolve_write_bytes() else { @@ -122,11 +126,7 @@ impl CliLockfile { maybe_deno_json: Option<&ConfigFile>, ) -> HashSet { maybe_deno_json - .map(|c| { - crate::args::deno_json::deno_json_deps(c) - .into_iter() - .collect() - }) + .map(crate::args::deno_json::deno_json_deps) .unwrap_or_default() } @@ -142,7 +142,7 @@ impl CliLockfile { return Ok(None); } - let filename = match flags.lock { + let file_path = match flags.lock { Some(ref lock) => PathBuf::from(lock), None => match workspace.resolve_lockfile_path()? { Some(path) => path, @@ -160,7 +160,11 @@ impl CliLockfile { .unwrap_or(false) }); - let lockfile = Self::read_from_path(filename, frozen)?; + let lockfile = Self::read_from_path(CliLockfileReadFromPathOptions { + file_path, + frozen, + skip_write: flags.internal.lockfile_skip_write, + })?; // initialize the lockfile with the workspace's configuration let root_url = workspace.root_dir(); @@ -212,25 +216,29 @@ impl CliLockfile { } pub fn read_from_path( - file_path: PathBuf, - frozen: bool, + opts: CliLockfileReadFromPathOptions, ) -> Result { - match std::fs::read_to_string(&file_path) { - Ok(text) => Ok(CliLockfile::new( - Lockfile::new(deno_lockfile::NewLockfileOptions { - file_path, - content: &text, - overwrite: false, - })?, - frozen, - )), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok( - CliLockfile::new(Lockfile::new_empty(file_path, false), frozen), - ), - Err(err) => Err(err).with_context(|| { - format!("Failed reading lockfile '{}'", file_path.display()) - }), - } + let lockfile = match std::fs::read_to_string(&opts.file_path) { + Ok(text) => Lockfile::new(deno_lockfile::NewLockfileOptions { + file_path: opts.file_path, + content: &text, + overwrite: false, + })?, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + Lockfile::new_empty(opts.file_path, false) + } + Err(err) => { + return Err(err).with_context(|| { + format!("Failed reading lockfile '{}'", opts.file_path.display()) + }); + } + }; + Ok(CliLockfile { + filename: lockfile.filename.clone(), + lockfile: Mutex::new(lockfile), + frozen: opts.frozen, + skip_write: opts.skip_write, + }) } pub fn error_if_changed(&self) -> Result<(), AnyError> { diff --git a/cli/args/mod.rs b/cli/args/mod.rs index 1c92777ae3e0cf..fb576a8c3e87e6 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -7,6 +7,7 @@ mod import_map; mod lockfile; mod package_json; +use deno_ast::MediaType; use deno_ast::SourceMapOption; use deno_config::deno_json::NodeModulesDirMode; use deno_config::workspace::CreateResolverOptions; @@ -20,20 +21,20 @@ use deno_config::workspace::WorkspaceDiscoverOptions; use deno_config::workspace::WorkspaceDiscoverStart; use deno_config::workspace::WorkspaceLintConfig; use deno_config::workspace::WorkspaceResolver; -use deno_core::normalize_path; use deno_core::resolve_url_or_path; use deno_graph::GraphKind; use deno_npm::npm_rc::NpmRc; use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmSystemInfo; +use deno_path_util::normalize_path; use deno_semver::npm::NpmPackageReqReference; +use deno_telemetry::OtelConfig; use import_map::resolve_import_map_value_from_specifier; pub use deno_config::deno_json::BenchConfig; pub use deno_config::deno_json::ConfigFile; pub use deno_config::deno_json::FmtOptionsConfig; -pub use deno_config::deno_json::JsxImportSourceConfig; pub use deno_config::deno_json::LintRulesConfig; pub use deno_config::deno_json::ProseWrap; pub use deno_config::deno_json::TsConfig; @@ -44,7 +45,9 @@ pub use deno_config::glob::FilePatterns; pub use deno_json::check_warn_tsconfig; pub use flags::*; pub use lockfile::CliLockfile; +pub use lockfile::CliLockfileReadFromPathOptions; pub use package_json::NpmInstallDepsProvider; +pub use package_json::PackageJsonDepValueParseWithLocationError; use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; @@ -69,6 +72,8 @@ use std::collections::HashMap; use std::env; use std::io::BufReader; use std::io::Cursor; +use std::io::Read; +use std::io::Seek; use std::net::SocketAddr; use std::num::NonZeroUsize; use std::path::Path; @@ -197,6 +202,8 @@ pub fn ts_config_to_transpile_and_emit_options( precompile_jsx_dynamic_props: None, transform_jsx, var_decl_imports: false, + // todo(dsherret): support verbatim_module_syntax here properly + verbatim_module_syntax: false, }, deno_ast::EmitOptions { inline_sources: options.inline_sources, @@ -282,6 +289,7 @@ impl BenchOptions { #[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] pub struct UnstableFmtOptions { pub component: bool, + pub sql: bool, } #[derive(Clone, Debug)] @@ -315,6 +323,7 @@ impl FmtOptions { options: resolve_fmt_options(fmt_flags, fmt_config.options), unstable: UnstableFmtOptions { component: unstable.component || fmt_flags.unstable_component, + sql: unstable.sql || fmt_flags.unstable_sql, }, files: fmt_config.files, } @@ -575,6 +584,7 @@ fn discover_npmrc( let resolved = npmrc .as_resolved(npm_registry_url()) .context("Failed to resolve .npmrc options")?; + log::debug!(".npmrc found at: '{}'", path.display()); Ok(Arc::new(resolved)) } @@ -742,15 +752,33 @@ pub enum NpmProcessStateKind { Byonm, } -pub(crate) const NPM_RESOLUTION_STATE_ENV_VAR_NAME: &str = - "DENO_DONT_USE_INTERNAL_NODE_COMPAT_STATE"; - static NPM_PROCESS_STATE: Lazy> = Lazy::new(|| { - let state = std::env::var(NPM_RESOLUTION_STATE_ENV_VAR_NAME).ok()?; - let state: NpmProcessState = serde_json::from_str(&state).ok()?; - // remove the environment variable so that sub processes - // that are spawned do not also use this. - std::env::remove_var(NPM_RESOLUTION_STATE_ENV_VAR_NAME); + use deno_runtime::ops::process::NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME; + let fd = std::env::var(NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME).ok()?; + std::env::remove_var(NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME); + let fd = fd.parse::().ok()?; + let mut file = { + use deno_runtime::deno_io::FromRawIoHandle; + unsafe { std::fs::File::from_raw_io_handle(fd as _) } + }; + let mut buf = Vec::new(); + // seek to beginning. after the file is written the position will be inherited by this subprocess, + // and also this file might have been read before + file.seek(std::io::SeekFrom::Start(0)).unwrap(); + file + .read_to_end(&mut buf) + .inspect_err(|e| { + log::error!("failed to read npm process state from fd {fd}: {e}"); + }) + .ok()?; + let state: NpmProcessState = serde_json::from_slice(&buf) + .inspect_err(|e| { + log::error!( + "failed to deserialize npm process state: {e} {}", + String::from_utf8_lossy(&buf) + ) + }) + .ok()?; Some(state) }); @@ -769,6 +797,7 @@ pub struct CliOptions { // application need not concern itself with, so keep these private flags: Arc, initial_cwd: PathBuf, + main_module_cell: std::sync::OnceLock>, maybe_node_modules_folder: Option, npmrc: Arc, maybe_lockfile: Option>, @@ -796,18 +825,14 @@ impl CliOptions { }; let msg = format!("DANGER: TLS certificate validation is disabled {}", domains); - #[allow(clippy::print_stderr)] { - // use eprintln instead of log::warn so this always gets shown - eprintln!("{}", colors::yellow(msg)); + log::error!("{}", colors::yellow(msg)); } } - warn_insecure_allow_run_flags(&flags); - let maybe_lockfile = maybe_lockfile.filter(|_| !force_global_cache); let deno_dir_provider = - Arc::new(DenoDirProvider::new(flags.cache_path.clone())); + Arc::new(DenoDirProvider::new(flags.internal.cache_path.clone())); let maybe_node_modules_folder = resolve_node_modules_folder( &initial_cwd, &flags, @@ -825,6 +850,7 @@ impl CliOptions { npmrc, maybe_node_modules_folder, overrides: Default::default(), + main_module_cell: std::sync::OnceLock::new(), start_dir, deno_dir_provider, }) @@ -844,12 +870,8 @@ impl CliOptions { } else { &[] }; - let config_parse_options = deno_config::deno_json::ConfigParseOptions { - include_task_comments: matches!( - flags.subcommand, - DenoSubcommand::Task(..) - ), - }; + let config_parse_options = + deno_config::deno_json::ConfigParseOptions::default(); let discover_pkg_json = flags.config_flag != ConfigFlag::Disabled && !flags.no_npm && !has_flag_env_var("DENO_NO_PACKAGE_JSON"); @@ -942,6 +964,9 @@ impl CliOptions { match self.sub_command() { DenoSubcommand::Cache(_) => GraphKind::All, DenoSubcommand::Check(_) => GraphKind::TypesOnly, + DenoSubcommand::Install(InstallFlags { + kind: InstallKind::Local(_), + }) => GraphKind::All, _ => self.type_check_mode().as_graph_kind(), } } @@ -1101,44 +1126,58 @@ impl CliOptions { } } - pub fn env_file_name(&self) -> Option<&String> { - self.flags.env_file.as_ref() + pub fn otel_config(&self) -> Option { + self.flags.otel_config() } - pub fn resolve_main_module(&self) -> Result { - let main_module = match &self.flags.subcommand { - DenoSubcommand::Compile(compile_flags) => { - resolve_url_or_path(&compile_flags.source_file, self.initial_cwd())? - } - DenoSubcommand::Eval(_) => { - resolve_url_or_path("./$deno$eval.ts", self.initial_cwd())? - } - DenoSubcommand::Repl(_) => { - resolve_url_or_path("./$deno$repl.ts", self.initial_cwd())? - } - DenoSubcommand::Run(run_flags) => { - if run_flags.is_stdin() { - std::env::current_dir() - .context("Unable to get CWD") - .and_then(|cwd| { - resolve_url_or_path("./$deno$stdin.ts", &cwd) - .map_err(AnyError::from) - })? - } else if NpmPackageReqReference::from_str(&run_flags.script).is_ok() { - ModuleSpecifier::parse(&run_flags.script)? - } else { - resolve_url_or_path(&run_flags.script, self.initial_cwd())? - } - } - DenoSubcommand::Serve(run_flags) => { - resolve_url_or_path(&run_flags.script, self.initial_cwd())? - } - _ => { - bail!("No main module.") - } - }; + pub fn env_file_name(&self) -> Option<&Vec> { + self.flags.env_file.as_ref() + } - Ok(main_module) + pub fn resolve_main_module(&self) -> Result<&ModuleSpecifier, AnyError> { + self + .main_module_cell + .get_or_init(|| { + Ok(match &self.flags.subcommand { + DenoSubcommand::Compile(compile_flags) => { + resolve_url_or_path(&compile_flags.source_file, self.initial_cwd())? + } + DenoSubcommand::Eval(_) => { + resolve_url_or_path("./$deno$eval.mts", self.initial_cwd())? + } + DenoSubcommand::Repl(_) => { + resolve_url_or_path("./$deno$repl.mts", self.initial_cwd())? + } + DenoSubcommand::Run(run_flags) => { + if run_flags.is_stdin() { + resolve_url_or_path("./$deno$stdin.mts", self.initial_cwd())? + } else { + let url = + resolve_url_or_path(&run_flags.script, self.initial_cwd())?; + if self.is_node_main() + && url.scheme() == "file" + && MediaType::from_specifier(&url) == MediaType::Unknown + { + try_resolve_node_binary_main_entrypoint( + &run_flags.script, + self.initial_cwd(), + )? + .unwrap_or(url) + } else { + url + } + } + } + DenoSubcommand::Serve(run_flags) => { + resolve_url_or_path(&run_flags.script, self.initial_cwd())? + } + _ => { + bail!("No main module.") + } + }) + }) + .as_ref() + .map_err(|err| deno_core::anyhow::anyhow!("{}", err)) } pub fn resolve_file_header_overrides( @@ -1159,7 +1198,7 @@ impl CliOptions { (maybe_main_specifier, maybe_content_type) { HashMap::from([( - main_specifier, + main_specifier.clone(), HashMap::from([("content-type".to_string(), content_type.to_string())]), )]) } else { @@ -1184,7 +1223,7 @@ impl CliOptions { // This is triggered via a secret environment variable which is used // for functionality like child_process.fork. Users should NOT depend // on this functionality. - pub fn is_npm_main(&self) -> bool { + pub fn is_node_main(&self) -> bool { NPM_PROCESS_STATE.is_some() } @@ -1282,6 +1321,7 @@ impl CliOptions { let workspace = self.workspace(); UnstableFmtOptions { component: workspace.has_unstable("fmt-component"), + sql: workspace.has_unstable("fmt-sql"), } } @@ -1322,11 +1362,9 @@ impl CliOptions { )?; Ok(deno_lint::linter::LintConfig { - default_jsx_factory: transpile_options - .jsx_automatic + default_jsx_factory: (!transpile_options.jsx_automatic) .then(|| transpile_options.jsx_factory.clone()), - default_jsx_fragment_factory: transpile_options - .jsx_automatic + default_jsx_fragment_factory: (!transpile_options.jsx_automatic) .then(|| transpile_options.jsx_fragment_factory.clone()), }) } @@ -1430,6 +1468,12 @@ impl CliOptions { watch: Some(WatchFlagsWithPaths { hmr, .. }), .. }) = &self.flags.subcommand + { + *hmr + } else if let DenoSubcommand::Serve(ServeFlags { + watch: Some(WatchFlagsWithPaths { hmr, .. }), + .. + }) = &self.flags.subcommand { *hmr } else { @@ -1480,7 +1524,38 @@ impl CliOptions { } pub fn permissions_options(&self) -> PermissionsOptions { - self.flags.permissions.to_options() + fn files_to_urls(files: &[String]) -> Vec> { + files + .iter() + .filter_map(|f| Url::parse(f).ok().map(Cow::Owned)) + .collect() + } + + // get a list of urls to imply for --allow-import + let cli_arg_urls = self + .resolve_main_module() + .ok() + .map(|url| vec![Cow::Borrowed(url)]) + .or_else(|| match &self.flags.subcommand { + DenoSubcommand::Cache(cache_flags) => { + Some(files_to_urls(&cache_flags.files)) + } + DenoSubcommand::Check(check_flags) => { + Some(files_to_urls(&check_flags.files)) + } + DenoSubcommand::Install(InstallFlags { + kind: InstallKind::Global(flags), + }) => Url::parse(&flags.module_url) + .ok() + .map(|url| vec![Cow::Owned(url)]), + DenoSubcommand::Doc(DocFlags { + source_files: DocSourceFileFlag::Paths(paths), + .. + }) => Some(files_to_urls(paths)), + _ => None, + }) + .unwrap_or_default(); + self.flags.permissions.to_options(&cli_arg_urls) } pub fn reload_flag(&self) -> bool { @@ -1531,6 +1606,18 @@ impl CliOptions { || self.workspace().has_unstable("bare-node-builtins") } + pub fn unstable_detect_cjs(&self) -> bool { + self.flags.unstable_config.detect_cjs + || self.workspace().has_unstable("detect-cjs") + } + + pub fn detect_cjs(&self) -> bool { + // only enabled when there's a package.json in order to not have a + // perf penalty for non-npm Deno projects of searching for the closest + // package.json beside each module + self.workspace().package_jsons().next().is_some() || self.is_node_main() + } + fn byonm_enabled(&self) -> bool { // check if enabled via unstable self.node_modules_dir().ok().flatten() == Some(NodeModulesDirMode::Manual) @@ -1541,6 +1628,17 @@ impl CliOptions { } pub fn use_byonm(&self) -> bool { + if matches!( + self.sub_command(), + DenoSubcommand::Install(_) + | DenoSubcommand::Add(_) + | DenoSubcommand::Remove(_) + | DenoSubcommand::Init(_) + | DenoSubcommand::Outdated(_) + ) { + // For `deno install/add/remove/init` we want to force the managed resolver so it can set up `node_modules/` directory. + return false; + } if self.node_modules_dir().ok().flatten().is_none() && self.maybe_node_modules_folder.is_some() && self @@ -1575,21 +1673,18 @@ impl CliOptions { }); if !from_config_file.is_empty() { - // collect unstable granular flags - let mut all_valid_unstable_flags: Vec<&str> = - crate::UNSTABLE_GRANULAR_FLAGS - .iter() - .map(|granular_flag| granular_flag.name) - .collect(); - - let mut another_unstable_flags = Vec::from([ - "sloppy-imports", - "byonm", - "bare-node-builtins", - "fmt-component", - ]); - // add more unstable flags to the same vector holding granular flags - all_valid_unstable_flags.append(&mut another_unstable_flags); + let all_valid_unstable_flags: Vec<&str> = crate::UNSTABLE_GRANULAR_FLAGS + .iter() + .map(|granular_flag| granular_flag.name) + .chain([ + "sloppy-imports", + "byonm", + "bare-node-builtins", + "detect-cjs", + "fmt-component", + "fmt-sql", + ]) + .collect(); // check and warn if the unstable flag of config file isn't supported, by // iterating through the vector holding the unstable flags @@ -1622,6 +1717,10 @@ impl CliOptions { if let DenoSubcommand::Run(RunFlags { watch: Some(WatchFlagsWithPaths { paths, .. }), .. + }) + | DenoSubcommand::Serve(ServeFlags { + watch: Some(WatchFlagsWithPaths { paths, .. }), + .. }) = &self.flags.subcommand { full_paths.extend(paths.iter().map(|path| self.initial_cwd.join(path))); @@ -1654,31 +1753,16 @@ impl CliOptions { allowed: self.flags.allow_scripts.clone(), initial_cwd: self.initial_cwd.clone(), root_dir: self.workspace().root_dir_path(), + explicit_install: matches!( + self.sub_command(), + DenoSubcommand::Install(_) + | DenoSubcommand::Cache(_) + | DenoSubcommand::Add(_) + ), } } } -/// Warns for specific uses of `--allow-run`. This function is not -/// intended to catch every single possible insecure use of `--allow-run`, -/// but is just an attempt to discourage some common pitfalls. -fn warn_insecure_allow_run_flags(flags: &Flags) { - let permissions = &flags.permissions; - if permissions.allow_all { - return; - } - let Some(allow_run_list) = permissions.allow_run.as_ref() else { - return; - }; - - // discourage using --allow-run without an allow list - if allow_run_list.is_empty() { - log::warn!( - "{} --allow-run without an allow list is susceptible to exploits. Prefer specifying an allow list (https://docs.deno.com/runtime/fundamentals/security/#running-subprocesses)", - colors::yellow("Warning") - ); - } -} - /// Resolves the path to use for a local node_modules folder. fn resolve_node_modules_folder( cwd: &Path, @@ -1736,6 +1820,36 @@ fn resolve_node_modules_folder( Ok(Some(canonicalize_path_maybe_not_exists(&path)?)) } +fn try_resolve_node_binary_main_entrypoint( + specifier: &str, + initial_cwd: &Path, +) -> Result, AnyError> { + // node allows running files at paths without a `.js` extension + // or at directories with an index.js file + let path = deno_core::normalize_path(initial_cwd.join(specifier)); + if path.is_dir() { + let index_file = path.join("index.js"); + Ok(if index_file.is_file() { + Some(deno_path_util::url_from_file_path(&index_file)?) + } else { + None + }) + } else { + let path = path.with_extension( + path + .extension() + .and_then(|s| s.to_str()) + .map(|s| format!("{}.js", s)) + .unwrap_or("js".to_string()), + ); + if path.is_file() { + Ok(Some(deno_path_util::url_from_file_path(&path)?)) + } else { + Ok(None) + } + } +} + fn resolve_import_map_specifier( maybe_import_map_path: Option<&str>, maybe_config_file: Option<&ConfigFile>, @@ -1806,6 +1920,10 @@ pub fn resolve_no_prompt(flags: &PermissionFlags) -> bool { flags.no_prompt || has_flag_env_var("DENO_NO_PROMPT") } +pub fn has_trace_permissions_enabled() -> bool { + has_flag_env_var("DENO_TRACE_PERMISSIONS") +} + pub fn has_flag_env_var(name: &str) -> bool { let value = env::var(name); matches!(value.as_ref().map(|s| s.as_str()), Ok("1")) @@ -1837,19 +1955,22 @@ pub fn config_to_deno_graph_workspace_member( }) } -fn load_env_variables_from_env_file(filename: Option<&String>) { - let Some(env_file_name) = filename else { +fn load_env_variables_from_env_file(filename: Option<&Vec>) { + let Some(env_file_names) = filename else { return; }; - match from_filename(env_file_name) { - Ok(_) => (), - Err(error) => { - match error { + + for env_file_name in env_file_names.iter().rev() { + match from_filename(env_file_name) { + Ok(_) => (), + Err(error) => { + match error { dotenvy::Error::LineParse(line, index)=> log::info!("{} Parsing failed within the specified environment file: {} at index: {} of the value: {}",colors::yellow("Warning"), env_file_name, index, line), dotenvy::Error::Io(_)=> log::info!("{} The `--env-file` flag was used, but the environment file specified '{}' was not found.",colors::yellow("Warning"),env_file_name), dotenvy::Error::EnvVar(_)=> log::info!("{} One or more of the environment variables isn't present or not unicode within the specified environment file: {}",colors::yellow("Warning"),env_file_name), _ => log::info!("{} Unknown failure occurred with the specified environment file: {}", colors::yellow("Warning"), env_file_name), } + } } } } diff --git a/cli/args/package_json.rs b/cli/args/package_json.rs index 2529e54fdfa7ff..7dc75550c31a97 100644 --- a/cli/args/package_json.rs +++ b/cli/args/package_json.rs @@ -5,9 +5,12 @@ use std::sync::Arc; use deno_config::workspace::Workspace; use deno_core::serde_json; +use deno_core::url::Url; use deno_package_json::PackageJsonDepValue; +use deno_package_json::PackageJsonDepValueParseError; use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageReq; +use thiserror::Error; #[derive(Debug)] pub struct InstallNpmRemotePkg { @@ -22,10 +25,20 @@ pub struct InstallNpmWorkspacePkg { pub target_dir: PathBuf, } +#[derive(Debug, Error, Clone)] +#[error("Failed to install '{}'\n at {}", alias, location)] +pub struct PackageJsonDepValueParseWithLocationError { + pub location: Url, + pub alias: String, + #[source] + pub source: PackageJsonDepValueParseError, +} + #[derive(Debug, Default)] pub struct NpmInstallDepsProvider { remote_pkgs: Vec, workspace_pkgs: Vec, + pkg_json_dep_errors: Vec, } impl NpmInstallDepsProvider { @@ -37,6 +50,7 @@ impl NpmInstallDepsProvider { // todo(dsherret): estimate capacity? let mut workspace_pkgs = Vec::new(); let mut remote_pkgs = Vec::new(); + let mut pkg_json_dep_errors = Vec::new(); let workspace_npm_pkgs = workspace.npm_packages(); for (_, folder) in workspace.config_folders() { @@ -83,8 +97,18 @@ impl NpmInstallDepsProvider { let deps = pkg_json.resolve_local_package_json_deps(); let mut pkg_pkgs = Vec::with_capacity(deps.len()); for (alias, dep) in deps { - let Ok(dep) = dep else { - continue; + let dep = match dep { + Ok(dep) => dep, + Err(err) => { + pkg_json_dep_errors.push( + PackageJsonDepValueParseWithLocationError { + location: pkg_json.specifier(), + alias, + source: err, + }, + ); + continue; + } }; match dep { PackageJsonDepValue::Req(pkg_req) => { @@ -131,14 +155,21 @@ impl NpmInstallDepsProvider { Self { remote_pkgs, workspace_pkgs, + pkg_json_dep_errors, } } - pub fn remote_pkgs(&self) -> &Vec { + pub fn remote_pkgs(&self) -> &[InstallNpmRemotePkg] { &self.remote_pkgs } - pub fn workspace_pkgs(&self) -> &Vec { + pub fn workspace_pkgs(&self) -> &[InstallNpmWorkspacePkg] { &self.workspace_pkgs } + + pub fn pkg_json_dep_errors( + &self, + ) -> &[PackageJsonDepValueParseWithLocationError] { + &self.pkg_json_dep_errors + } } diff --git a/cli/bench/encode_into.js b/cli/bench/encode_into.js index 11f5a56d9024a5..ab5e11b04db847 100644 --- a/cli/bench/encode_into.js +++ b/cli/bench/encode_into.js @@ -1,6 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -// deno-lint-ignore-file no-console +// deno-lint-ignore-file no-console no-process-globals let [total, count] = typeof Deno !== "undefined" ? Deno.args diff --git a/cli/bench/getrandom.js b/cli/bench/getrandom.js index 3c3ec4aa19aaa3..fe99bbcbdf7c91 100644 --- a/cli/bench/getrandom.js +++ b/cli/bench/getrandom.js @@ -1,6 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -// deno-lint-ignore-file no-console +// deno-lint-ignore-file no-console no-process-globals let [total, count] = typeof Deno !== "undefined" ? Deno.args diff --git a/cli/bench/http.rs b/cli/bench/http.rs deleted file mode 100644 index f739b76ba83497..00000000000000 --- a/cli/bench/http.rs +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -use std::collections::HashMap; -use std::net::TcpStream; -use std::path::Path; -use std::process::Command; -use std::sync::atomic::AtomicU16; -use std::sync::atomic::Ordering; -use std::time::Duration; -use std::time::Instant; - -use super::Result; - -pub use test_util::parse_wrk_output; -pub use test_util::WrkOutput as HttpBenchmarkResult; -// Some of the benchmarks in this file have been renamed. In case the history -// somehow gets messed up: -// "node_http" was once called "node" -// "deno_tcp" was once called "deno" -// "deno_http" was once called "deno_net_http" - -const DURATION: &str = "10s"; - -pub fn benchmark( - target_path: &Path, -) -> Result> { - let deno_exe = test_util::deno_exe_path(); - let deno_exe = deno_exe.to_string(); - - let hyper_hello_exe = target_path.join("test_server"); - let hyper_hello_exe = hyper_hello_exe.to_str().unwrap(); - - let mut res = HashMap::new(); - let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); - let http_dir = manifest_dir.join("bench").join("http"); - for entry in std::fs::read_dir(&http_dir)? { - let entry = entry?; - let pathbuf = entry.path(); - let path = pathbuf.to_str().unwrap(); - if path.ends_with(".lua") { - continue; - } - let file_stem = pathbuf.file_stem().unwrap().to_str().unwrap(); - - let lua_script = http_dir.join(format!("{file_stem}.lua")); - let mut maybe_lua = None; - if lua_script.exists() { - maybe_lua = Some(lua_script.to_str().unwrap()); - } - - let port = get_port(); - // deno run -A --unstable-net - res.insert( - file_stem.to_string(), - run( - &[ - deno_exe.as_str(), - "run", - "--allow-all", - "--unstable-net", - "--enable-testing-features-do-not-use", - path, - &server_addr(port), - ], - port, - None, - None, - maybe_lua, - )?, - ); - } - - res.insert("hyper".to_string(), hyper_http(hyper_hello_exe)?); - - Ok(res) -} - -fn run( - server_cmd: &[&str], - port: u16, - env: Option>, - origin_cmd: Option<&[&str]>, - lua_script: Option<&str>, -) -> Result { - // Wait for port 4544 to become available. - // TODO Need to use SO_REUSEPORT with tokio::net::TcpListener. - std::thread::sleep(Duration::from_secs(5)); - - let mut origin = None; - if let Some(cmd) = origin_cmd { - let mut com = Command::new(cmd[0]); - com.args(&cmd[1..]); - if let Some(env) = env.clone() { - com.envs(env); - } - origin = Some(com.spawn()?); - }; - - println!("{}", server_cmd.join(" ")); - let mut server = { - let mut com = Command::new(server_cmd[0]); - com.args(&server_cmd[1..]); - if let Some(env) = env { - com.envs(env); - } - com.spawn()? - }; - - // Wait for server to wake up. - let now = Instant::now(); - let addr = format!("127.0.0.1:{port}"); - while now.elapsed().as_secs() < 30 { - if TcpStream::connect(&addr).is_ok() { - break; - } - std::thread::sleep(Duration::from_millis(10)); - } - TcpStream::connect(&addr).expect("Failed to connect to server in time"); - println!("Server took {} ms to start", now.elapsed().as_millis()); - - let wrk = test_util::prebuilt_tool_path("wrk"); - assert!(wrk.is_file()); - - let addr = format!("http://{addr}/"); - let wrk = wrk.to_string(); - let mut wrk_cmd = vec![wrk.as_str(), "-d", DURATION, "--latency", &addr]; - - if let Some(lua_script) = lua_script { - wrk_cmd.push("-s"); - wrk_cmd.push(lua_script); - } - - println!("{}", wrk_cmd.join(" ")); - let output = test_util::run_collect(&wrk_cmd, None, None, None, true).0; - - std::thread::sleep(Duration::from_secs(1)); // wait to capture failure. TODO racy. - - println!("{output}"); - assert!( - server.try_wait()?.map(|s| s.success()).unwrap_or(true), - "server ended with error" - ); - - server.kill()?; - if let Some(mut origin) = origin { - origin.kill()?; - } - - Ok(parse_wrk_output(&output)) -} - -static NEXT_PORT: AtomicU16 = AtomicU16::new(4544); -pub(crate) fn get_port() -> u16 { - let p = NEXT_PORT.load(Ordering::SeqCst); - NEXT_PORT.store(p.wrapping_add(1), Ordering::SeqCst); - p -} - -fn server_addr(port: u16) -> String { - format!("0.0.0.0:{port}") -} - -fn hyper_http(exe: &str) -> Result { - let port = get_port(); - println!("http_benchmark testing RUST hyper"); - run(&[exe, &port.to_string()], port, None, None, None) -} diff --git a/cli/bench/http/deno_flash_hono_router.js b/cli/bench/http/deno_flash_hono_router.js deleted file mode 100644 index baced0cecee8f5..00000000000000 --- a/cli/bench/http/deno_flash_hono_router.js +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import { Hono } from "https://deno.land/x/hono@v2.0.9/mod.ts"; - -const addr = Deno.args[0] || "127.0.0.1:4500"; -const [hostname, port] = addr.split(":"); - -const app = new Hono(); -app.get("/", (c) => c.text("Hello, World!")); - -Deno.serve({ port: Number(port), hostname }, app.fetch); diff --git a/cli/bench/http/deno_flash_send_file.js b/cli/bench/http/deno_flash_send_file.js deleted file mode 100644 index bf8541f8b0d1e5..00000000000000 --- a/cli/bench/http/deno_flash_send_file.js +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -const addr = Deno.args[0] || "127.0.0.1:4500"; -const [hostname, port] = addr.split(":"); -const { serve } = Deno; - -const path = new URL("../testdata/128k.bin", import.meta.url).pathname; - -function handler() { - const file = Deno.openSync(path); - return new Response(file.readable); -} - -serve({ hostname, port: Number(port) }, handler); diff --git a/cli/bench/http/deno_http_read_headers.lua b/cli/bench/http/deno_http_read_headers.lua deleted file mode 100644 index 64f1923ff3ee15..00000000000000 --- a/cli/bench/http/deno_http_read_headers.lua +++ /dev/null @@ -1,5 +0,0 @@ -wrk.headers["foo"] = "bar" -wrk.headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36" -wrk.headers["Viewport-Width"] = "1920" -wrk.headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" -wrk.headers["Accept-Language"] = "en,la;q=0.9" \ No newline at end of file diff --git a/cli/bench/http/deno_http_serve.js b/cli/bench/http/deno_http_serve.js deleted file mode 100644 index 639982ce606bdf..00000000000000 --- a/cli/bench/http/deno_http_serve.js +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -const addr = Deno.args[0] ?? "127.0.0.1:4500"; -const [hostname, port] = addr.split(":"); -const { serve } = Deno; - -function handler() { - return new Response("Hello World"); -} - -serve({ hostname, port: Number(port), reusePort: true }, handler); diff --git a/cli/bench/http/deno_post_bin.lua b/cli/bench/http/deno_post_bin.lua deleted file mode 100644 index c8f5d3e3f7e9df..00000000000000 --- a/cli/bench/http/deno_post_bin.lua +++ /dev/null @@ -1,5 +0,0 @@ -wrk.method = "POST" -wrk.headers["Content-Type"] = "application/octet-stream" - -file = io.open("./cli/bench/testdata/128k.bin", "rb") -wrk.body = file:read("*a") \ No newline at end of file diff --git a/cli/bench/http/deno_post_json.lua b/cli/bench/http/deno_post_json.lua deleted file mode 100644 index cc6c4e226d0206..00000000000000 --- a/cli/bench/http/deno_post_json.lua +++ /dev/null @@ -1,3 +0,0 @@ -wrk.method = "POST" -wrk.headers["Content-Type"] = "application/json" -wrk.body = '{"hello":"deno"}' \ No newline at end of file diff --git a/cli/bench/http/deno_reactdom_ssr_flash.jsx b/cli/bench/http/deno_reactdom_ssr_flash.jsx deleted file mode 100644 index eaabf891219535..00000000000000 --- a/cli/bench/http/deno_reactdom_ssr_flash.jsx +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -import { renderToReadableStream } from "https://esm.run/react-dom/server"; -import * as React from "https://esm.run/react"; -const { serve } = Deno; -const addr = Deno.args[0] || "127.0.0.1:4500"; -const [hostname, port] = addr.split(":"); - -const App = () => ( - - -

Hello World

- - -); - -const headers = { - headers: { - "Content-Type": "text/html", - }, -}; - -serve({ hostname, port: Number(port) }, async () => { - return new Response(await renderToReadableStream(), headers); -}); diff --git a/cli/bench/http/deno_tcp.ts b/cli/bench/http/deno_tcp.ts deleted file mode 100644 index b7959107374786..00000000000000 --- a/cli/bench/http/deno_tcp.ts +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -// Used for benchmarking Deno's networking. -// TODO(bartlomieju): Replace this with a real HTTP server once -// https://github.com/denoland/deno/issues/726 is completed. -// Note: this is a keep-alive server. -// deno-lint-ignore-file no-console -const addr = Deno.args[0] || "127.0.0.1:4500"; -const [hostname, port] = addr.split(":"); -const listener = Deno.listen({ hostname, port: Number(port) }); -const response = new TextEncoder().encode( - "HTTP/1.1 200 OK\r\nContent-Length: 12\r\n\r\nHello World\n", -); -async function handle(conn: Deno.Conn): Promise { - const buffer = new Uint8Array(1024); - try { - while (true) { - await conn.read(buffer); - await conn.write(response); - } - } catch (e) { - if ( - !(e instanceof Deno.errors.BrokenPipe) && - !(e instanceof Deno.errors.ConnectionReset) - ) { - throw e; - } - } - conn.close(); -} - -console.log("Listening on", addr); -for await (const conn of listener) { - handle(conn); -} diff --git a/cli/bench/lsp.rs b/cli/bench/lsp.rs index b088865c6b2e70..7baaffca7eb969 100644 --- a/cli/bench/lsp.rs +++ b/cli/bench/lsp.rs @@ -150,7 +150,11 @@ fn bench_big_file_edits(deno_exe: &Path) -> Duration { .deno_exe(deno_exe) .build(); client.initialize_default(); + let (method, _): (String, Option) = client.read_notification(); + assert_eq!(method, "deno/didRefreshDenoConfigurationTree"); client.change_configuration(json!({ "deno": { "enable": true } })); + let (method, _): (String, Option) = client.read_notification(); + assert_eq!(method, "deno/didRefreshDenoConfigurationTree"); client.write_notification( "textDocument/didOpen", @@ -206,6 +210,8 @@ fn bench_code_lens(deno_exe: &Path) -> Duration { .deno_exe(deno_exe) .build(); client.initialize_default(); + let (method, _): (String, Option) = client.read_notification(); + assert_eq!(method, "deno/didRefreshDenoConfigurationTree"); client.change_configuration(json!({ "deno": { "enable": true, "codeLens": { @@ -214,6 +220,8 @@ fn bench_code_lens(deno_exe: &Path) -> Duration { "test": true, }, } })); + let (method, _): (String, Option) = client.read_notification(); + assert_eq!(method, "deno/didRefreshDenoConfigurationTree"); client.write_notification( "textDocument/didOpen", @@ -257,7 +265,11 @@ fn bench_find_replace(deno_exe: &Path) -> Duration { .deno_exe(deno_exe) .build(); client.initialize_default(); + let (method, _): (String, Option) = client.read_notification(); + assert_eq!(method, "deno/didRefreshDenoConfigurationTree"); client.change_configuration(json!({ "deno": { "enable": true } })); + let (method, _): (String, Option) = client.read_notification(); + assert_eq!(method, "deno/didRefreshDenoConfigurationTree"); for i in 0..10 { client.write_notification( @@ -341,7 +353,11 @@ fn bench_startup_shutdown(deno_exe: &Path) -> Duration { .deno_exe(deno_exe) .build(); client.initialize_default(); + let (method, _): (String, Option) = client.read_notification(); + assert_eq!(method, "deno/didRefreshDenoConfigurationTree"); client.change_configuration(json!({ "deno": { "enable": true } })); + let (method, _): (String, Option) = client.read_notification(); + assert_eq!(method, "deno/didRefreshDenoConfigurationTree"); client.write_notification( "textDocument/didOpen", diff --git a/cli/bench/lsp_bench_standalone.rs b/cli/bench/lsp_bench_standalone.rs index 9c4f264ec9d42f..3c946cfbe30403 100644 --- a/cli/bench/lsp_bench_standalone.rs +++ b/cli/bench/lsp_bench_standalone.rs @@ -13,7 +13,11 @@ use test_util::lsp::LspClientBuilder; fn incremental_change_wait(bench: &mut Bencher) { let mut client = LspClientBuilder::new().use_diagnostic_sync(false).build(); client.initialize_default(); + let (method, _): (String, Option) = client.read_notification(); + assert_eq!(method, "deno/didRefreshDenoConfigurationTree"); client.change_configuration(json!({ "deno": { "enable": true } })); + let (method, _): (String, Option) = client.read_notification(); + assert_eq!(method, "deno/didRefreshDenoConfigurationTree"); client.write_notification( "textDocument/didOpen", diff --git a/cli/bench/main.rs b/cli/bench/main.rs index 72fa7e96366fec..c3c42d24882f04 100644 --- a/cli/bench/main.rs +++ b/cli/bench/main.rs @@ -17,7 +17,6 @@ use std::process::Stdio; use std::time::SystemTime; use test_util::PathRef; -mod http; mod lsp; fn read_json(filename: &Path) -> Result { @@ -345,9 +344,11 @@ struct BenchResult { binary_size: HashMap, bundle_size: HashMap, cargo_deps: usize, + // TODO(bartlomieju): remove max_latency: HashMap, max_memory: HashMap, lsp_exec_time: HashMap, + // TODO(bartlomieju): remove req_per_sec: HashMap, syscall_count: HashMap, thread_count: HashMap, @@ -362,7 +363,6 @@ async fn main() -> Result<()> { "binary_size", "cargo_deps", "lsp", - "http", "strace", "mem_usage", ]; @@ -427,21 +427,6 @@ async fn main() -> Result<()> { new_data.lsp_exec_time = lsp_exec_times; } - if benchmarks.contains(&"http") && cfg!(not(target_os = "windows")) { - let stats = http::benchmark(target_dir.as_path())?; - let req_per_sec = stats - .iter() - .map(|(name, result)| (name.clone(), result.requests as i64)) - .collect(); - new_data.req_per_sec = req_per_sec; - let max_latency = stats - .iter() - .map(|(name, result)| (name.clone(), result.latency)) - .collect(); - - new_data.max_latency = max_latency; - } - if cfg!(target_os = "linux") && benchmarks.contains(&"strace") { use std::io::Read; diff --git a/cli/bench/op_now.js b/cli/bench/op_now.js index bcc3ea3c568442..7c1427c8096b6b 100644 --- a/cli/bench/op_now.js +++ b/cli/bench/op_now.js @@ -1,6 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -// deno-lint-ignore-file no-console +// deno-lint-ignore-file no-console no-process-globals const queueMicrotask = globalThis.queueMicrotask || process.nextTick; let [total, count] = typeof Deno !== "undefined" diff --git a/cli/bench/secure_curves.js b/cli/bench/secure_curves.js index 02d248b23febf3..912b75cccd8c0b 100644 --- a/cli/bench/secure_curves.js +++ b/cli/bench/secure_curves.js @@ -1,6 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -// deno-lint-ignore-file no-console +// deno-lint-ignore-file no-console no-process-globals let [total, count] = typeof Deno !== "undefined" ? Deno.args diff --git a/cli/bench/tty.js b/cli/bench/tty.js index 248a9011375beb..e494e76af72354 100644 --- a/cli/bench/tty.js +++ b/cli/bench/tty.js @@ -1,6 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -// deno-lint-ignore-file no-console +// deno-lint-ignore-file no-console no-process-globals const queueMicrotask = globalThis.queueMicrotask || process.nextTick; let [total, count] = typeof Deno !== "undefined" diff --git a/cli/bench/url_parse.js b/cli/bench/url_parse.js index 367cf73f460763..9cb0045f644e25 100644 --- a/cli/bench/url_parse.js +++ b/cli/bench/url_parse.js @@ -1,6 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -// deno-lint-ignore-file no-console +// deno-lint-ignore-file no-console no-process-globals const queueMicrotask = globalThis.queueMicrotask || process.nextTick; let [total, count] = typeof Deno !== "undefined" diff --git a/cli/bench/write_file.js b/cli/bench/write_file.js index 104a23a8dbd1ca..747503ce2a10f8 100644 --- a/cli/bench/write_file.js +++ b/cli/bench/write_file.js @@ -1,6 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -// deno-lint-ignore-file no-console +// deno-lint-ignore-file no-console no-process-globals const queueMicrotask = globalThis.queueMicrotask || process.nextTick; let [total, count] = typeof Deno !== "undefined" diff --git a/cli/build.rs b/cli/build.rs index c3b6f8b0454bc9..3d986612841584 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -365,6 +365,9 @@ fn main() { return; } + deno_napi::print_linker_flags("deno"); + deno_napi::print_linker_flags("denort"); + // Host snapshots won't work when cross compiling. let target = env::var("TARGET").unwrap(); let host = env::var("HOST").unwrap(); @@ -374,56 +377,6 @@ fn main() { panic!("Cross compiling with snapshot is not supported."); } - let symbols_file_name = match env::consts::OS { - "android" | "freebsd" | "openbsd" => { - "generated_symbol_exports_list_linux.def".to_string() - } - os => format!("generated_symbol_exports_list_{}.def", os), - }; - let symbols_path = std::path::Path::new("napi") - .join(symbols_file_name) - .canonicalize() - .expect( - "Missing symbols list! Generate using tools/napi/generate_symbols_lists.js", - ); - - #[cfg(target_os = "windows")] - println!( - "cargo:rustc-link-arg-bin=deno=/DEF:{}", - symbols_path.display() - ); - - #[cfg(target_os = "macos")] - println!( - "cargo:rustc-link-arg-bin=deno=-Wl,-exported_symbols_list,{}", - symbols_path.display() - ); - - #[cfg(target_os = "linux")] - { - // If a custom compiler is set, the glibc version is not reliable. - // Here, we assume that if a custom compiler is used, that it will be modern enough to support a dynamic symbol list. - if env::var("CC").is_err() - && glibc_version::get_version() - .map(|ver| ver.major <= 2 && ver.minor < 35) - .unwrap_or(false) - { - println!("cargo:warning=Compiling with all symbols exported, this will result in a larger binary. Please use glibc 2.35 or later for an optimised build."); - println!("cargo:rustc-link-arg-bin=deno=-rdynamic"); - } else { - println!( - "cargo:rustc-link-arg-bin=deno=-Wl,--export-dynamic-symbol-list={}", - symbols_path.display() - ); - } - } - - #[cfg(target_os = "android")] - println!( - "cargo:rustc-link-arg-bin=deno=-Wl,--export-dynamic-symbol-list={}", - symbols_path.display() - ); - // To debug snapshot issues uncomment: // op_fetch_asset::trace_serializer(); @@ -447,6 +400,24 @@ fn main() { println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap()); println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); + if cfg!(windows) { + // these dls load slowly, so delay loading them + let dlls = [ + // webgpu + "d3dcompiler_47", + "OPENGL32", + // network related functions + "iphlpapi", + ]; + for dll in dlls { + println!("cargo:rustc-link-arg-bin=deno=/delayload:{dll}.dll"); + println!("cargo:rustc-link-arg-bin=denort=/delayload:{dll}.dll"); + } + // enable delay loading + println!("cargo:rustc-link-arg-bin=deno=delayimp.lib"); + println!("cargo:rustc-link-arg-bin=denort=delayimp.lib"); + } + let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); let o = PathBuf::from(env::var_os("OUT_DIR").unwrap()); diff --git a/cli/cache/cache_db.rs b/cli/cache/cache_db.rs index b24078f29bd6fd..329ed2d9704dff 100644 --- a/cli/cache/cache_db.rs +++ b/cli/cache/cache_db.rs @@ -57,7 +57,7 @@ impl rusqlite::types::FromSql for CacheDBHash { } /// What should the cache should do on failure? -#[derive(Default)] +#[derive(Debug, Default)] pub enum CacheFailure { /// Return errors if failure mode otherwise unspecified. #[default] @@ -69,6 +69,7 @@ pub enum CacheFailure { } /// Configuration SQL and other parameters for a [`CacheDB`]. +#[derive(Debug)] pub struct CacheDBConfiguration { /// SQL to run for a new database. pub table_initializer: &'static str, @@ -98,6 +99,7 @@ impl CacheDBConfiguration { } } +#[derive(Debug)] enum ConnectionState { Connected(Connection), Blackhole, @@ -106,7 +108,7 @@ enum ConnectionState { /// A cache database that eagerly initializes itself off-thread, preventing initialization operations /// from blocking the main thread. -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct CacheDB { // TODO(mmastrac): We can probably simplify our thread-safe implementation here conn: Arc>>, diff --git a/cli/cache/code_cache.rs b/cli/cache/code_cache.rs index abcd0d46ac1cff..b1d9ae757b9038 100644 --- a/cli/cache/code_cache.rs +++ b/cli/cache/code_cache.rs @@ -1,10 +1,14 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::sync::Arc; + use deno_ast::ModuleSpecifier; use deno_core::error::AnyError; use deno_runtime::code_cache; use deno_runtime::deno_webstorage::rusqlite::params; +use crate::worker::CliCodeCache; + use super::cache_db::CacheDB; use super::cache_db::CacheDBConfiguration; use super::cache_db::CacheDBHash; @@ -82,6 +86,12 @@ impl CodeCache { } } +impl CliCodeCache for CodeCache { + fn as_code_cache(self: Arc) -> Arc { + self + } +} + impl code_cache::CodeCache for CodeCache { fn get_sync( &self, diff --git a/cli/cache/deno_dir.rs b/cli/cache/deno_dir.rs index 88d8a31c04fc63..7b7059c224df3c 100644 --- a/cli/cache/deno_dir.rs +++ b/cli/cache/deno_dir.rs @@ -126,9 +126,9 @@ impl DenoDir { self.root.join("registries") } - /// Path to the dependencies cache folder. - pub fn deps_folder_path(&self) -> PathBuf { - self.root.join("deps") + /// Path to the remote cache folder. + pub fn remote_folder_path(&self) -> PathBuf { + self.root.join("remote") } /// Path to the origin data cache folder. diff --git a/cli/cache/emit.rs b/cli/cache/emit.rs index 6807f06c1063e4..3c9eecfcbdb6b2 100644 --- a/cli/cache/emit.rs +++ b/cli/cache/emit.rs @@ -10,6 +10,7 @@ use deno_core::unsync::sync::AtomicFlag; use super::DiskCache; /// The cache that stores previously emitted files. +#[derive(Debug)] pub struct EmitCache { disk_cache: DiskCache, emit_failed_flag: AtomicFlag, @@ -39,7 +40,7 @@ impl EmitCache { &self, specifier: &ModuleSpecifier, expected_source_hash: u64, - ) -> Option> { + ) -> Option { let emit_filename = self.get_emit_filename(specifier)?; let bytes = self.disk_cache.get(&emit_filename).ok()?; self @@ -91,6 +92,7 @@ impl EmitCache { const LAST_LINE_PREFIX: &str = "\n// denoCacheMetadata="; +#[derive(Debug)] struct EmitFileSerializer { cli_version: &'static str, } @@ -100,7 +102,7 @@ impl EmitFileSerializer { &self, mut bytes: Vec, expected_source_hash: u64, - ) -> Option> { + ) -> Option { let last_newline_index = bytes.iter().rposition(|&b| b == b'\n')?; let (content, last_line) = bytes.split_at(last_newline_index); let hashes = last_line.strip_prefix(LAST_LINE_PREFIX.as_bytes())?; @@ -120,7 +122,7 @@ impl EmitFileSerializer { // everything looks good, truncate and return it bytes.truncate(content.len()); - Some(bytes) + String::from_utf8(bytes).ok() } pub fn serialize(&self, code: &[u8], source_hash: u64) -> Vec { @@ -170,8 +172,6 @@ mod test { }, emit_failed_flag: Default::default(), }; - let to_string = - |bytes: Vec| -> String { String::from_utf8(bytes).unwrap() }; let specifier1 = ModuleSpecifier::from_file_path(temp_dir.path().join("file1.ts")) @@ -188,13 +188,10 @@ mod test { assert_eq!(cache.get_emit_code(&specifier1, 5), None); // providing the correct source hash assert_eq!( - cache.get_emit_code(&specifier1, 10).map(to_string), + cache.get_emit_code(&specifier1, 10), Some(emit_code1.clone()), ); - assert_eq!( - cache.get_emit_code(&specifier2, 2).map(to_string), - Some(emit_code2) - ); + assert_eq!(cache.get_emit_code(&specifier2, 2), Some(emit_code2)); // try changing the cli version (should not load previous ones) let cache = EmitCache { @@ -215,18 +212,12 @@ mod test { }, emit_failed_flag: Default::default(), }; - assert_eq!( - cache.get_emit_code(&specifier1, 5).map(to_string), - Some(emit_code1) - ); + assert_eq!(cache.get_emit_code(&specifier1, 5), Some(emit_code1)); // adding when already exists should not cause issue let emit_code3 = "asdf".to_string(); cache.set_emit_code(&specifier1, 20, emit_code3.as_bytes()); assert_eq!(cache.get_emit_code(&specifier1, 5), None); - assert_eq!( - cache.get_emit_code(&specifier1, 20).map(to_string), - Some(emit_code3) - ); + assert_eq!(cache.get_emit_code(&specifier1, 20), Some(emit_code3)); } } diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index a95c3508661ce9..73a3895a10205f 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -1,15 +1,16 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use crate::args::jsr_url; use crate::args::CacheSetting; use crate::errors::get_error_class_name; use crate::file_fetcher::FetchNoFollowOptions; use crate::file_fetcher::FetchOptions; -use crate::file_fetcher::FetchPermissionsOption; +use crate::file_fetcher::FetchPermissionsOptionRef; use crate::file_fetcher::FileFetcher; use crate::file_fetcher::FileOrRedirect; -use crate::npm::CliNpmResolver; use crate::util::fs::atomic_write_file_with_retries; -use crate::util::path::specifier_has_extension; +use crate::util::fs::atomic_write_file_with_retries_and_fs; +use crate::util::fs::AtomicWriteFileFsAdapter; use deno_ast::MediaType; use deno_core::futures; @@ -19,6 +20,9 @@ use deno_graph::source::CacheInfo; use deno_graph::source::LoadFuture; use deno_graph::source::LoadResponse; use deno_graph::source::Loader; +use deno_runtime::deno_fs; +use deno_runtime::deno_permissions::PermissionsContainer; +use node_resolver::InNpmPackageChecker; use std::collections::HashMap; use std::path::Path; use std::path::PathBuf; @@ -75,8 +79,12 @@ impl deno_cache_dir::DenoCacheEnv for RealDenoCacheEnv { atomic_write_file_with_retries(path, bytes, CACHE_PERM) } - fn remove_file(&self, path: &Path) -> std::io::Result<()> { - std::fs::remove_file(path) + fn canonicalize_path(&self, path: &Path) -> std::io::Result { + crate::util::fs::canonicalize_path(path) + } + + fn create_dir_all(&self, path: &Path) -> std::io::Result<()> { + std::fs::create_dir_all(path) } fn modified(&self, path: &Path) -> std::io::Result> { @@ -98,40 +106,113 @@ impl deno_cache_dir::DenoCacheEnv for RealDenoCacheEnv { } } +#[derive(Debug, Clone)] +pub struct DenoCacheEnvFsAdapter<'a>( + pub &'a dyn deno_runtime::deno_fs::FileSystem, +); + +impl<'a> deno_cache_dir::DenoCacheEnv for DenoCacheEnvFsAdapter<'a> { + fn read_file_bytes(&self, path: &Path) -> std::io::Result> { + self + .0 + .read_file_sync(path, None) + // todo(https://github.com/denoland/deno_cache_dir/pull/66): avoid clone + .map(|bytes| bytes.into_owned()) + .map_err(|err| err.into_io_error()) + } + + fn atomic_write_file( + &self, + path: &Path, + bytes: &[u8], + ) -> std::io::Result<()> { + atomic_write_file_with_retries_and_fs( + &AtomicWriteFileFsAdapter { + fs: self.0, + write_mode: CACHE_PERM, + }, + path, + bytes, + ) + } + + fn canonicalize_path(&self, path: &Path) -> std::io::Result { + self.0.realpath_sync(path).map_err(|e| e.into_io_error()) + } + + fn create_dir_all(&self, path: &Path) -> std::io::Result<()> { + self + .0 + .mkdir_sync(path, true, None) + .map_err(|e| e.into_io_error()) + } + + fn modified(&self, path: &Path) -> std::io::Result> { + self + .0 + .stat_sync(path) + .map(|stat| { + stat + .mtime + .map(|ts| SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(ts)) + }) + .map_err(|e| e.into_io_error()) + } + + fn is_file(&self, path: &Path) -> bool { + self.0.is_file_sync(path) + } + + fn time_now(&self) -> SystemTime { + SystemTime::now() + } +} + pub type GlobalHttpCache = deno_cache_dir::GlobalHttpCache; pub type LocalHttpCache = deno_cache_dir::LocalHttpCache; pub type LocalLspHttpCache = deno_cache_dir::LocalLspHttpCache; pub use deno_cache_dir::HttpCache; +pub struct FetchCacherOptions { + pub file_header_overrides: HashMap>, + pub permissions: PermissionsContainer, + /// If we're publishing for `deno publish`. + pub is_deno_publish: bool, +} + /// A "wrapper" for the FileFetcher and DiskCache for the Deno CLI that provides /// a concise interface to the DENO_DIR when building module graphs. pub struct FetchCacher { - file_fetcher: Arc, pub file_header_overrides: HashMap>, + file_fetcher: Arc, + fs: Arc, global_http_cache: Arc, - npm_resolver: Arc, + in_npm_pkg_checker: Arc, module_info_cache: Arc, - permissions: FetchPermissionsOption, + permissions: PermissionsContainer, + is_deno_publish: bool, cache_info_enabled: bool, } impl FetchCacher { pub fn new( file_fetcher: Arc, - file_header_overrides: HashMap>, + fs: Arc, global_http_cache: Arc, - npm_resolver: Arc, + in_npm_pkg_checker: Arc, module_info_cache: Arc, - permissions: FetchPermissionsOption, + options: FetchCacherOptions, ) -> Self { Self { file_fetcher, - file_header_overrides, + fs, global_http_cache, - npm_resolver, + in_npm_pkg_checker, module_info_cache, - permissions, + file_header_overrides: options.file_header_overrides, + permissions: options.permissions, + is_deno_publish: options.is_deno_publish, cache_info_enabled: false, } } @@ -182,36 +263,43 @@ impl Loader for FetchCacher { ) -> LoadFuture { use deno_graph::source::CacheSetting as LoaderCacheSetting; - if specifier.scheme() == "file" { - if specifier.path().contains("/node_modules/") { - // The specifier might be in a completely different symlinked tree than - // what the node_modules url is in (ex. `/my-project-1/node_modules` - // symlinked to `/my-project-2/node_modules`), so first we checked if the path - // is in a node_modules dir to avoid needlessly canonicalizing, then now compare - // against the canonicalized specifier. - let specifier = - crate::node::resolve_specifier_into_node_modules(specifier); - if self.npm_resolver.in_npm_package(&specifier) { - return Box::pin(futures::future::ready(Ok(Some( - LoadResponse::External { specifier }, - )))); - } - } - - // make local CJS modules external to the graph - if specifier_has_extension(specifier, "cjs") { + if specifier.scheme() == "file" + && specifier.path().contains("/node_modules/") + { + // The specifier might be in a completely different symlinked tree than + // what the node_modules url is in (ex. `/my-project-1/node_modules` + // symlinked to `/my-project-2/node_modules`), so first we checked if the path + // is in a node_modules dir to avoid needlessly canonicalizing, then now compare + // against the canonicalized specifier. + let specifier = crate::node::resolve_specifier_into_node_modules( + specifier, + self.fs.as_ref(), + ); + if self.in_npm_pkg_checker.in_npm_package(&specifier) { return Box::pin(futures::future::ready(Ok(Some( - LoadResponse::External { - specifier: specifier.clone(), - }, + LoadResponse::External { specifier }, )))); } } + if self.is_deno_publish + && matches!(specifier.scheme(), "http" | "https") + && !specifier.as_str().starts_with(jsr_url().as_str()) + { + // mark non-JSR remote modules as external so we don't need --allow-import + // permissions as these will error out later when publishing + return Box::pin(futures::future::ready(Ok(Some( + LoadResponse::External { + specifier: specifier.clone(), + }, + )))); + } + let file_fetcher = self.file_fetcher.clone(); let file_header_overrides = self.file_header_overrides.clone(); let permissions = self.permissions.clone(); let specifier = specifier.clone(); + let is_statically_analyzable = !options.was_dynamic_root; async move { let maybe_cache_setting = match options.cache_setting { @@ -230,7 +318,12 @@ impl Loader for FetchCacher { .fetch_no_follow_with_options(FetchNoFollowOptions { fetch_options: FetchOptions { specifier: &specifier, - permissions: permissions.as_ref(), + permissions: if is_statically_analyzable { + FetchPermissionsOptionRef::StaticContainer(&permissions) + } else { + FetchPermissionsOptionRef::DynamicContainer(&permissions) + }, + maybe_auth: None, maybe_accept: None, maybe_cache_setting: maybe_cache_setting.as_ref(), }, diff --git a/cli/cache/module_info.rs b/cli/cache/module_info.rs index 4dbb01c37b1bf1..469e2fafac3732 100644 --- a/cli/cache/module_info.rs +++ b/cli/cache/module_info.rs @@ -44,18 +44,32 @@ pub static MODULE_INFO_CACHE_DB: CacheDBConfiguration = CacheDBConfiguration { /// A cache of `deno_graph::ModuleInfo` objects. Using this leads to a considerable /// performance improvement because when it exists we can skip parsing a module for /// deno_graph. +#[derive(Debug)] pub struct ModuleInfoCache { conn: CacheDB, + parsed_source_cache: Arc, } impl ModuleInfoCache { #[cfg(test)] - pub fn new_in_memory(version: &'static str) -> Self { - Self::new(CacheDB::in_memory(&MODULE_INFO_CACHE_DB, version)) + pub fn new_in_memory( + version: &'static str, + parsed_source_cache: Arc, + ) -> Self { + Self::new( + CacheDB::in_memory(&MODULE_INFO_CACHE_DB, version), + parsed_source_cache, + ) } - pub fn new(conn: CacheDB) -> Self { - Self { conn } + pub fn new( + conn: CacheDB, + parsed_source_cache: Arc, + ) -> Self { + Self { + conn, + parsed_source_cache, + } } /// Useful for testing: re-create this cache DB with a different current version. @@ -63,6 +77,7 @@ impl ModuleInfoCache { pub(crate) fn recreate_with_version(self, version: &'static str) -> Self { Self { conn: self.conn.recreate_with_version(version), + parsed_source_cache: self.parsed_source_cache, } } @@ -113,13 +128,10 @@ impl ModuleInfoCache { Ok(()) } - pub fn as_module_analyzer<'a>( - &'a self, - parsed_source_cache: &'a Arc, - ) -> ModuleInfoCacheModuleAnalyzer<'a> { + pub fn as_module_analyzer(&self) -> ModuleInfoCacheModuleAnalyzer { ModuleInfoCacheModuleAnalyzer { module_info_cache: self, - parsed_source_cache, + parsed_source_cache: &self.parsed_source_cache, } } } @@ -129,31 +141,99 @@ pub struct ModuleInfoCacheModuleAnalyzer<'a> { parsed_source_cache: &'a Arc, } -#[async_trait::async_trait(?Send)] -impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> { - async fn analyze( +impl<'a> ModuleInfoCacheModuleAnalyzer<'a> { + fn load_cached_module_info( &self, specifier: &ModuleSpecifier, - source: Arc, media_type: MediaType, - ) -> Result { - // attempt to load from the cache - let source_hash = CacheDBHash::from_source(&source); + source_hash: CacheDBHash, + ) -> Option { match self.module_info_cache.get_module_info( specifier, media_type, source_hash, ) { - Ok(Some(info)) => return Ok(info), - Ok(None) => {} + Ok(Some(info)) => Some(info), + Ok(None) => None, Err(err) => { log::debug!( "Error loading module cache info for {}. {:#}", specifier, err ); + None } } + } + + fn save_module_info_to_cache( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + source_hash: CacheDBHash, + module_info: &ModuleInfo, + ) { + if let Err(err) = self.module_info_cache.set_module_info( + specifier, + media_type, + source_hash, + module_info, + ) { + log::debug!( + "Error saving module cache info for {}. {:#}", + specifier, + err + ); + } + } + + pub fn analyze_sync( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + source: &Arc, + ) -> Result { + // attempt to load from the cache + let source_hash = CacheDBHash::from_source(source); + if let Some(info) = + self.load_cached_module_info(specifier, media_type, source_hash) + { + return Ok(info); + } + + // otherwise, get the module info from the parsed source cache + let parser = self.parsed_source_cache.as_capturing_parser(); + let analyzer = ParserModuleAnalyzer::new(&parser); + let module_info = + analyzer.analyze_sync(specifier, source.clone(), media_type)?; + + // then attempt to cache it + self.save_module_info_to_cache( + specifier, + media_type, + source_hash, + &module_info, + ); + + Ok(module_info) + } +} + +#[async_trait::async_trait(?Send)] +impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> { + async fn analyze( + &self, + specifier: &ModuleSpecifier, + source: Arc, + media_type: MediaType, + ) -> Result { + // attempt to load from the cache + let source_hash = CacheDBHash::from_source(&source); + if let Some(info) = + self.load_cached_module_info(specifier, media_type, source_hash) + { + return Ok(info); + } // otherwise, get the module info from the parsed source cache let module_info = deno_core::unsync::spawn_blocking({ @@ -169,18 +249,12 @@ impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> { .unwrap()?; // then attempt to cache it - if let Err(err) = self.module_info_cache.set_module_info( + self.save_module_info_to_cache( specifier, media_type, source_hash, &module_info, - ) { - log::debug!( - "Error saving module cache info for {}. {:#}", - specifier, - err - ); - } + ); Ok(module_info) } @@ -202,7 +276,7 @@ fn serialize_media_type(media_type: MediaType) -> i64 { Tsx => 11, Json => 12, Wasm => 13, - TsBuildInfo => 14, + Css => 14, SourceMap => 15, Unknown => 16, } @@ -210,6 +284,7 @@ fn serialize_media_type(media_type: MediaType) -> i64 { #[cfg(test)] mod test { + use deno_graph::JsDocImportInfo; use deno_graph::PositionRange; use deno_graph::SpecifierWithRange; @@ -217,7 +292,7 @@ mod test { #[test] pub fn module_info_cache_general_use() { - let cache = ModuleInfoCache::new_in_memory("1.0.0"); + let cache = ModuleInfoCache::new_in_memory("1.0.0", Default::default()); let specifier1 = ModuleSpecifier::parse("https://localhost/mod.ts").unwrap(); let specifier2 = @@ -234,18 +309,21 @@ mod test { ); let mut module_info = ModuleInfo::default(); - module_info.jsdoc_imports.push(SpecifierWithRange { - range: PositionRange { - start: deno_graph::Position { - line: 0, - character: 3, - }, - end: deno_graph::Position { - line: 1, - character: 2, + module_info.jsdoc_imports.push(JsDocImportInfo { + specifier: SpecifierWithRange { + range: PositionRange { + start: deno_graph::Position { + line: 0, + character: 3, + }, + end: deno_graph::Position { + line: 1, + character: 2, + }, }, + text: "test".to_string(), }, - text: "test".to_string(), + resolution_mode: None, }); cache .set_module_info( diff --git a/cli/cache/parsed_source.rs b/cli/cache/parsed_source.rs index e956361f461532..7e819ae9985ba3 100644 --- a/cli/cache/parsed_source.rs +++ b/cli/cache/parsed_source.rs @@ -7,9 +7,9 @@ use deno_ast::MediaType; use deno_ast::ModuleSpecifier; use deno_ast::ParsedSource; use deno_core::parking_lot::Mutex; -use deno_graph::CapturingModuleParser; -use deno_graph::DefaultModuleParser; -use deno_graph::ModuleParser; +use deno_graph::CapturingEsParser; +use deno_graph::DefaultEsParser; +use deno_graph::EsParser; use deno_graph::ParseOptions; use deno_graph::ParsedSourceStore; @@ -46,7 +46,7 @@ impl<'a> LazyGraphSourceParser<'a> { } } -#[derive(Default)] +#[derive(Debug, Default)] pub struct ParsedSourceCache { sources: Mutex>, } @@ -57,12 +57,11 @@ impl ParsedSourceCache { module: &deno_graph::JsModule, ) -> Result { let parser = self.as_capturing_parser(); - // this will conditionally parse because it's using a CapturingModuleParser - parser.parse_module(ParseOptions { + // this will conditionally parse because it's using a CapturingEsParser + parser.parse_program(ParseOptions { specifier: &module.specifier, source: module.source.clone(), media_type: module.media_type, - // don't bother enabling because this method is currently only used for vendoring scope_analysis: false, }) } @@ -86,10 +85,9 @@ impl ParsedSourceCache { specifier, source, media_type, - // don't bother enabling because this method is currently only used for emitting scope_analysis: false, }; - DefaultModuleParser.parse_module(options) + DefaultEsParser.parse_program(options) } /// Frees the parsed source from memory. @@ -99,8 +97,8 @@ impl ParsedSourceCache { /// Creates a parser that will reuse a ParsedSource from the store /// if it exists, or else parse. - pub fn as_capturing_parser(&self) -> CapturingModuleParser { - CapturingModuleParser::new(None, self) + pub fn as_capturing_parser(&self) -> CapturingEsParser { + CapturingEsParser::new(None, self) } } diff --git a/cli/clippy.toml b/cli/clippy.toml index e20c56c47abba3..f1c25acfb87da6 100644 --- a/cli/clippy.toml +++ b/cli/clippy.toml @@ -1,5 +1,6 @@ disallowed-methods = [ { path = "reqwest::Client::new", reason = "create an HttpClient via an HttpClientProvider instead" }, + { path = "std::process::exit", reason = "use deno_runtime::exit instead" }, ] disallowed-types = [ { path = "reqwest::Client", reason = "use crate::http_util::HttpClient instead" }, diff --git a/cli/emit.rs b/cli/emit.rs index b3f4a4477aed94..3cd23b7abbcaa6 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -3,24 +3,28 @@ use crate::cache::EmitCache; use crate::cache::FastInsecureHasher; use crate::cache::ParsedSourceCache; +use crate::resolver::CjsTracker; +use deno_ast::ModuleKind; use deno_ast::SourceMapOption; use deno_ast::SourceRange; use deno_ast::SourceRanged; use deno_ast::SourceRangedForSpanned; +use deno_ast::TranspileModuleOptions; use deno_ast::TranspileResult; use deno_core::error::AnyError; use deno_core::futures::stream::FuturesUnordered; use deno_core::futures::FutureExt; use deno_core::futures::StreamExt; -use deno_core::ModuleCodeBytes; use deno_core::ModuleSpecifier; use deno_graph::MediaType; use deno_graph::Module; use deno_graph::ModuleGraph; use std::sync::Arc; +#[derive(Debug)] pub struct Emitter { + cjs_tracker: Arc, emit_cache: Arc, parsed_source_cache: Arc, transpile_and_emit_options: @@ -31,6 +35,7 @@ pub struct Emitter { impl Emitter { pub fn new( + cjs_tracker: Arc, emit_cache: Arc, parsed_source_cache: Arc, transpile_options: deno_ast::TranspileOptions, @@ -43,6 +48,7 @@ impl Emitter { hasher.finish() }; Self { + cjs_tracker, emit_cache, parsed_source_cache, transpile_and_emit_options: Arc::new((transpile_options, emit_options)), @@ -60,20 +66,19 @@ impl Emitter { continue; }; - let is_emittable = matches!( - module.media_type, - MediaType::TypeScript - | MediaType::Mts - | MediaType::Cts - | MediaType::Jsx - | MediaType::Tsx - ); - if is_emittable { + if module.media_type.is_emittable() { futures.push( self .emit_parsed_source( &module.specifier, module.media_type, + ModuleKind::from_is_cjs( + self.cjs_tracker.is_cjs_with_known_is_script( + &module.specifier, + module.media_type, + module.is_script, + )?, + ), &module.source, ) .boxed_local(), @@ -92,9 +97,10 @@ impl Emitter { pub fn maybe_cached_emit( &self, specifier: &ModuleSpecifier, + module_kind: deno_ast::ModuleKind, source: &str, - ) -> Option> { - let source_hash = self.get_source_hash(source); + ) -> Option { + let source_hash = self.get_source_hash(module_kind, source); self.emit_cache.get_emit_code(specifier, source_hash) } @@ -102,25 +108,27 @@ impl Emitter { &self, specifier: &ModuleSpecifier, media_type: MediaType, + module_kind: deno_ast::ModuleKind, source: &Arc, - ) -> Result { + ) -> Result { // Note: keep this in sync with the sync version below let helper = EmitParsedSourceHelper(self); - match helper.pre_emit_parsed_source(specifier, source) { + match helper.pre_emit_parsed_source(specifier, module_kind, source) { PreEmitResult::Cached(emitted_text) => Ok(emitted_text), PreEmitResult::NotCached { source_hash } => { let parsed_source_cache = self.parsed_source_cache.clone(); let transpile_and_emit_options = self.transpile_and_emit_options.clone(); - let transpile_result = deno_core::unsync::spawn_blocking({ + let transpiled_source = deno_core::unsync::spawn_blocking({ let specifier = specifier.clone(); let source = source.clone(); move || -> Result<_, AnyError> { EmitParsedSourceHelper::transpile( &parsed_source_cache, &specifier, - source.clone(), media_type, + module_kind, + source.clone(), &transpile_and_emit_options.0, &transpile_and_emit_options.1, ) @@ -128,11 +136,12 @@ impl Emitter { }) .await .unwrap()?; - Ok(helper.post_emit_parsed_source( + helper.post_emit_parsed_source( specifier, - transpile_result, + &transpiled_source, source_hash, - )) + ); + Ok(transpiled_source) } } } @@ -141,26 +150,29 @@ impl Emitter { &self, specifier: &ModuleSpecifier, media_type: MediaType, + module_kind: deno_ast::ModuleKind, source: &Arc, - ) -> Result { + ) -> Result { // Note: keep this in sync with the async version above let helper = EmitParsedSourceHelper(self); - match helper.pre_emit_parsed_source(specifier, source) { + match helper.pre_emit_parsed_source(specifier, module_kind, source) { PreEmitResult::Cached(emitted_text) => Ok(emitted_text), PreEmitResult::NotCached { source_hash } => { - let transpile_result = EmitParsedSourceHelper::transpile( + let transpiled_source = EmitParsedSourceHelper::transpile( &self.parsed_source_cache, specifier, - source.clone(), media_type, + module_kind, + source.clone(), &self.transpile_and_emit_options.0, &self.transpile_and_emit_options.1, )?; - Ok(helper.post_emit_parsed_source( + helper.post_emit_parsed_source( specifier, - transpile_result, + &transpiled_source, source_hash, - )) + ); + Ok(transpiled_source) } } } @@ -190,10 +202,20 @@ impl Emitter { // this statement is probably wrong) let mut options = self.transpile_and_emit_options.1.clone(); options.source_map = SourceMapOption::None; + let is_cjs = self.cjs_tracker.is_cjs_with_known_is_script( + specifier, + media_type, + parsed_source.compute_is_script(), + )?; let transpiled_source = parsed_source - .transpile(&self.transpile_and_emit_options.0, &options)? - .into_source() - .into_string()?; + .transpile( + &self.transpile_and_emit_options.0, + &deno_ast::TranspileModuleOptions { + module_kind: Some(ModuleKind::from_is_cjs(is_cjs)), + }, + &options, + )? + .into_source(); Ok(transpiled_source.text) } MediaType::JavaScript @@ -204,7 +226,7 @@ impl Emitter { | MediaType::Dcts | MediaType::Json | MediaType::Wasm - | MediaType::TsBuildInfo + | MediaType::Css | MediaType::SourceMap | MediaType::Unknown => { // clear this specifier from the parsed source cache as it's now out of date @@ -217,16 +239,17 @@ impl Emitter { /// A hashing function that takes the source code and uses the global emit /// options then generates a string hash which can be stored to /// determine if the cached emit is valid or not. - fn get_source_hash(&self, source_text: &str) -> u64 { + fn get_source_hash(&self, module_kind: ModuleKind, source_text: &str) -> u64 { FastInsecureHasher::new_without_deno_version() // stored in the transpile_and_emit_options_hash .write_str(source_text) .write_u64(self.transpile_and_emit_options_hash) + .write_hashable(module_kind) .finish() } } enum PreEmitResult { - Cached(ModuleCodeBytes), + Cached(String), NotCached { source_hash: u64 }, } @@ -237,14 +260,15 @@ impl<'a> EmitParsedSourceHelper<'a> { pub fn pre_emit_parsed_source( &self, specifier: &ModuleSpecifier, + module_kind: deno_ast::ModuleKind, source: &Arc, ) -> PreEmitResult { - let source_hash = self.0.get_source_hash(source); + let source_hash = self.0.get_source_hash(module_kind, source); if let Some(emit_code) = self.0.emit_cache.get_emit_code(specifier, source_hash) { - PreEmitResult::Cached(emit_code.into_boxed_slice().into()) + PreEmitResult::Cached(emit_code) } else { PreEmitResult::NotCached { source_hash } } @@ -253,25 +277,24 @@ impl<'a> EmitParsedSourceHelper<'a> { pub fn transpile( parsed_source_cache: &ParsedSourceCache, specifier: &ModuleSpecifier, - source: Arc, media_type: MediaType, + module_kind: deno_ast::ModuleKind, + source: Arc, transpile_options: &deno_ast::TranspileOptions, emit_options: &deno_ast::EmitOptions, - ) -> Result { + ) -> Result { // nothing else needs the parsed source at this point, so remove from // the cache in order to not transpile owned let parsed_source = parsed_source_cache .remove_or_parse_module(specifier, source, media_type)?; ensure_no_import_assertion(&parsed_source)?; - Ok(parsed_source.transpile(transpile_options, emit_options)?) - } - - pub fn post_emit_parsed_source( - &self, - specifier: &ModuleSpecifier, - transpile_result: TranspileResult, - source_hash: u64, - ) -> ModuleCodeBytes { + let transpile_result = parsed_source.transpile( + transpile_options, + &TranspileModuleOptions { + module_kind: Some(module_kind), + }, + emit_options, + )?; let transpiled_source = match transpile_result { TranspileResult::Owned(source) => source, TranspileResult::Cloned(source) => { @@ -280,12 +303,20 @@ impl<'a> EmitParsedSourceHelper<'a> { } }; debug_assert!(transpiled_source.source_map.is_none()); + Ok(transpiled_source.text) + } + + pub fn post_emit_parsed_source( + &self, + specifier: &ModuleSpecifier, + transpiled_source: &str, + source_hash: u64, + ) { self.0.emit_cache.set_emit_code( specifier, source_hash, - &transpiled_source.source, + transpiled_source.as_bytes(), ); - transpiled_source.source.into_boxed_slice().into() } } @@ -317,7 +348,7 @@ fn ensure_no_import_assertion( deno_core::anyhow::anyhow!("{}", msg) } - let Some(module) = parsed_source.program_ref().as_module() else { + let deno_ast::ProgramRef::Module(module) = parsed_source.program_ref() else { return Ok(()); }; diff --git a/cli/errors.rs b/cli/errors.rs index 25b3fc33243806..38dc8259e3c61c 100644 --- a/cli/errors.rs +++ b/cli/errors.rs @@ -38,6 +38,7 @@ fn get_module_graph_error_class(err: &ModuleGraphError) -> &'static str { ModuleGraphError::ModuleError(err) => match err { ModuleError::InvalidTypeAssertion { .. } => "SyntaxError", ModuleError::ParseErr(_, diagnostic) => get_diagnostic_class(diagnostic), + ModuleError::WasmParseErr(..) => "SyntaxError", ModuleError::UnsupportedMediaType { .. } | ModuleError::UnsupportedImportAttributeType { .. } => "TypeError", ModuleError::Missing(_, _) | ModuleError::MissingDynamic(_, _) => { @@ -88,6 +89,10 @@ fn get_resolution_error_class(err: &ResolutionError) -> &'static str { } } +fn get_try_from_int_error_class(_: &std::num::TryFromIntError) -> &'static str { + "TypeError" +} + pub fn get_error_class_name(e: &AnyError) -> &'static str { deno_runtime::errors::get_error_class_name(e) .or_else(|| { @@ -106,5 +111,9 @@ pub fn get_error_class_name(e: &AnyError) -> &'static str { e.downcast_ref::() .map(get_resolution_error_class) }) + .or_else(|| { + e.downcast_ref::() + .map(get_try_from_int_error_class) + }) .unwrap_or("Error") } diff --git a/cli/factory.rs b/cli/factory.rs index 0f49546d07f742..5d9a2c0824152e 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -11,6 +11,7 @@ use crate::args::StorageKeyResolver; use crate::args::TsConfigType; use crate::cache::Caches; use crate::cache::CodeCache; +use crate::cache::DenoCacheEnvFsAdapter; use crate::cache::DenoDir; use crate::cache::DenoDirProvider; use crate::cache::EmitCache; @@ -32,22 +33,29 @@ use crate::module_loader::ModuleLoadPreparer; use crate::node::CliCjsCodeAnalyzer; use crate::node::CliNodeCodeTranslator; use crate::npm::create_cli_npm_resolver; +use crate::npm::create_in_npm_pkg_checker; +use crate::npm::CliByonmNpmResolverCreateOptions; +use crate::npm::CliManagedInNpmPkgCheckerCreateOptions; +use crate::npm::CliManagedNpmResolverCreateOptions; use crate::npm::CliNpmResolver; -use crate::npm::CliNpmResolverByonmCreateOptions; use crate::npm::CliNpmResolverCreateOptions; -use crate::npm::CliNpmResolverManagedCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; -use crate::resolver::CjsResolutionStore; -use crate::resolver::CliGraphResolver; -use crate::resolver::CliGraphResolverOptions; -use crate::resolver::CliNodeResolver; +use crate::npm::CreateInNpmPkgCheckerOptions; +use crate::resolver::CjsTracker; +use crate::resolver::CliDenoResolver; +use crate::resolver::CliDenoResolverFs; +use crate::resolver::CliNpmReqResolver; +use crate::resolver::CliResolver; +use crate::resolver::CliResolverOptions; +use crate::resolver::CliSloppyImportsResolver; use crate::resolver::NpmModuleLoader; -use crate::resolver::SloppyImportsResolver; +use crate::resolver::SloppyImportsCachedFs; use crate::standalone::DenoCompileBinaryWriter; use crate::tools::check::TypeChecker; use crate::tools::coverage::CoverageCollector; use crate::tools::lint::LintRuleProvider; use crate::tools::run::hmr::HmrRunner; +use crate::tsc::TypeCheckingCjsTracker; use crate::util::file_watcher::WatcherCommunicator; use crate::util::fs::canonicalize_path_maybe_not_exists; use crate::util::progress_bar::ProgressBar; @@ -56,15 +64,21 @@ use crate::worker::CliMainWorkerFactory; use crate::worker::CliMainWorkerOptions; use std::path::PathBuf; +use deno_cache_dir::npm::NpmCacheDir; use deno_config::workspace::PackageJsonDepResolution; use deno_config::workspace::WorkspaceResolver; use deno_core::error::AnyError; use deno_core::futures::FutureExt; use deno_core::FeatureChecker; +use deno_resolver::cjs::IsCjsResolutionMode; +use deno_resolver::npm::NpmReqResolverOptions; +use deno_resolver::DenoResolverOptions; +use deno_resolver::NodeAndNpmReqResolver; use deno_runtime::deno_fs; use deno_runtime::deno_node::DenoFsNodeResolverEnv; use deno_runtime::deno_node::NodeResolver; +use deno_runtime::deno_node::PackageJsonResolver; use deno_runtime::deno_permissions::Permissions; use deno_runtime::deno_permissions::PermissionsContainer; use deno_runtime::deno_tls::rustls::RootCertStore; @@ -74,6 +88,7 @@ use deno_runtime::inspector_server::InspectorServer; use deno_runtime::permissions::RuntimePermissionDescriptorParser; use log::warn; use node_resolver::analyze::NodeCodeTranslator; +use node_resolver::InNpmPackageChecker; use once_cell::sync::OnceCell; use std::future::Future; use std::sync::Arc; @@ -115,7 +130,7 @@ impl RootCertStoreProvider for CliRootCertStoreProvider { } } -struct Deferred(once_cell::unsync::OnceCell); +pub struct Deferred(once_cell::unsync::OnceCell); impl Default for Deferred { fn default() -> Self { @@ -161,37 +176,42 @@ impl Deferred { #[derive(Default)] struct CliFactoryServices { - cli_options: Deferred>, + blob_store: Deferred>, caches: Deferred>, + cjs_tracker: Deferred>, + cli_options: Deferred>, + code_cache: Deferred>, + deno_resolver: Deferred>, + emit_cache: Deferred>, + emitter: Deferred>, + feature_checker: Deferred>, file_fetcher: Deferred>, + fs: Deferred>, global_http_cache: Deferred>, http_cache: Deferred>, http_client_provider: Deferred>, - emit_cache: Deferred>, - emitter: Deferred>, - fs: Deferred>, + in_npm_pkg_checker: Deferred>, main_graph_container: Deferred>, - maybe_inspector_server: Deferred>>, - root_cert_store_provider: Deferred>, - blob_store: Deferred>, - module_info_cache: Deferred>, - parsed_source_cache: Deferred>, - resolver: Deferred>, maybe_file_watcher_reporter: Deferred>, + maybe_inspector_server: Deferred>>, module_graph_builder: Deferred>, module_graph_creator: Deferred>, + module_info_cache: Deferred>, module_load_preparer: Deferred>, node_code_translator: Deferred>, node_resolver: Deferred>, + npm_cache_dir: Deferred>, + npm_req_resolver: Deferred>, npm_resolver: Deferred>, + parsed_source_cache: Deferred>, permission_desc_parser: Deferred>, - sloppy_imports_resolver: Deferred>>, + pkg_json_resolver: Deferred>, + resolver: Deferred>, + root_cert_store_provider: Deferred>, + root_permissions_container: Deferred, + sloppy_imports_resolver: Deferred>>, text_only_progress_bar: Deferred, type_checker: Deferred>, - cjs_resolutions: Deferred>, - cli_node_resolver: Deferred>, - feature_checker: Deferred>, - code_cache: Deferred>, workspace_resolver: Deferred>, } @@ -298,7 +318,7 @@ impl CliFactory { pub fn global_http_cache(&self) -> Result<&Arc, AnyError> { self.services.global_http_cache.get_or_try_init(|| { Ok(Arc::new(GlobalHttpCache::new( - self.deno_dir()?.deps_folder_path(), + self.deno_dir()?.remote_folder_path(), crate::cache::RealDenoCacheEnv, ))) }) @@ -348,72 +368,127 @@ impl CliFactory { self.services.fs.get_or_init(|| Arc::new(deno_fs::RealFs)) } + pub fn in_npm_pkg_checker( + &self, + ) -> Result<&Arc, AnyError> { + self.services.in_npm_pkg_checker.get_or_try_init(|| { + let cli_options = self.cli_options()?; + let options = if cli_options.use_byonm() { + CreateInNpmPkgCheckerOptions::Byonm + } else { + CreateInNpmPkgCheckerOptions::Managed( + CliManagedInNpmPkgCheckerCreateOptions { + root_cache_dir_url: self.npm_cache_dir()?.root_dir_url(), + maybe_node_modules_path: cli_options + .node_modules_dir_path() + .map(|p| p.as_path()), + }, + ) + }; + Ok(create_in_npm_pkg_checker(options)) + }) + } + + pub fn npm_cache_dir(&self) -> Result<&Arc, AnyError> { + self.services.npm_cache_dir.get_or_try_init(|| { + let fs = self.fs(); + let global_path = self.deno_dir()?.npm_folder_path(); + let cli_options = self.cli_options()?; + Ok(Arc::new(NpmCacheDir::new( + &DenoCacheEnvFsAdapter(fs.as_ref()), + global_path, + cli_options.npmrc().get_all_known_registries_urls(), + ))) + }) + } + pub async fn npm_resolver( &self, ) -> Result<&Arc, AnyError> { self .services .npm_resolver - .get_or_try_init_async(async { - let fs = self.fs(); - let cli_options = self.cli_options()?; - // For `deno install` we want to force the managed resolver so it can set up `node_modules/` directory. - create_cli_npm_resolver(if cli_options.use_byonm() && !matches!(cli_options.sub_command(), DenoSubcommand::Install(_) | DenoSubcommand::Add(_) | DenoSubcommand::Remove(_)) { - CliNpmResolverCreateOptions::Byonm(CliNpmResolverByonmCreateOptions { - fs: fs.clone(), - root_node_modules_dir: Some(match cli_options.node_modules_dir_path() { - Some(node_modules_path) => node_modules_path.to_path_buf(), - // path needs to be canonicalized for node resolution - // (node_modules_dir_path above is already canonicalized) - None => canonicalize_path_maybe_not_exists(cli_options.initial_cwd())? - .join("node_modules"), - }), - }) - } else { - CliNpmResolverCreateOptions::Managed(CliNpmResolverManagedCreateOptions { - snapshot: match cli_options.resolve_npm_resolution_snapshot()? { - Some(snapshot) => { - CliNpmResolverManagedSnapshotOption::Specified(Some(snapshot)) - } - None => match cli_options.maybe_lockfile() { - Some(lockfile) => { - CliNpmResolverManagedSnapshotOption::ResolveFromLockfile( - lockfile.clone(), - ) - } - None => CliNpmResolverManagedSnapshotOption::Specified(None), + .get_or_try_init_async( + async { + let fs = self.fs(); + let cli_options = self.cli_options()?; + create_cli_npm_resolver(if cli_options.use_byonm() { + CliNpmResolverCreateOptions::Byonm( + CliByonmNpmResolverCreateOptions { + fs: CliDenoResolverFs(fs.clone()), + pkg_json_resolver: self.pkg_json_resolver().clone(), + root_node_modules_dir: Some( + match cli_options.node_modules_dir_path() { + Some(node_modules_path) => node_modules_path.to_path_buf(), + // path needs to be canonicalized for node resolution + // (node_modules_dir_path above is already canonicalized) + None => canonicalize_path_maybe_not_exists( + cli_options.initial_cwd(), + )? + .join("node_modules"), + }, + ), }, - }, - maybe_lockfile: cli_options.maybe_lockfile().cloned(), - fs: fs.clone(), - http_client_provider: self.http_client_provider().clone(), - npm_global_cache_dir: self.deno_dir()?.npm_folder_path(), - cache_setting: cli_options.cache_setting(), - text_only_progress_bar: self.text_only_progress_bar().clone(), - maybe_node_modules_path: cli_options.node_modules_dir_path().cloned(), - npm_install_deps_provider: Arc::new(NpmInstallDepsProvider::from_workspace(cli_options.workspace())), - npm_system_info: cli_options.npm_system_info(), - npmrc: cli_options.npmrc().clone(), - lifecycle_scripts: cli_options.lifecycle_scripts_config(), + ) + } else { + CliNpmResolverCreateOptions::Managed( + CliManagedNpmResolverCreateOptions { + snapshot: match cli_options.resolve_npm_resolution_snapshot()? { + Some(snapshot) => { + CliNpmResolverManagedSnapshotOption::Specified(Some( + snapshot, + )) + } + None => match cli_options.maybe_lockfile() { + Some(lockfile) => { + CliNpmResolverManagedSnapshotOption::ResolveFromLockfile( + lockfile.clone(), + ) + } + None => { + CliNpmResolverManagedSnapshotOption::Specified(None) + } + }, + }, + maybe_lockfile: cli_options.maybe_lockfile().cloned(), + fs: fs.clone(), + http_client_provider: self.http_client_provider().clone(), + npm_cache_dir: self.npm_cache_dir()?.clone(), + cache_setting: cli_options.cache_setting(), + text_only_progress_bar: self.text_only_progress_bar().clone(), + maybe_node_modules_path: cli_options + .node_modules_dir_path() + .cloned(), + npm_install_deps_provider: Arc::new( + NpmInstallDepsProvider::from_workspace( + cli_options.workspace(), + ), + ), + npm_system_info: cli_options.npm_system_info(), + npmrc: cli_options.npmrc().clone(), + lifecycle_scripts: cli_options.lifecycle_scripts_config(), + }, + ) }) - }).await - }.boxed_local()) + .await + } + .boxed_local(), + ) .await } pub fn sloppy_imports_resolver( &self, - ) -> Result>, AnyError> { + ) -> Result>, AnyError> { self .services .sloppy_imports_resolver .get_or_try_init(|| { - Ok( - self - .cli_options()? - .unstable_sloppy_imports() - .then(|| Arc::new(SloppyImportsResolver::new(self.fs().clone()))), - ) + Ok(self.cli_options()?.unstable_sloppy_imports().then(|| { + Arc::new(CliSloppyImportsResolver::new(SloppyImportsCachedFs::new( + self.fs().clone(), + ))) + })) }) .map(|maybe| maybe.as_ref()) } @@ -453,28 +528,47 @@ impl CliFactory { .await } - pub async fn resolver(&self) -> Result<&Arc, AnyError> { + pub async fn deno_resolver(&self) -> Result<&Arc, AnyError> { + self + .services + .deno_resolver + .get_or_try_init_async(async { + let cli_options = self.cli_options()?; + Ok(Arc::new(CliDenoResolver::new(DenoResolverOptions { + in_npm_pkg_checker: self.in_npm_pkg_checker()?.clone(), + node_and_req_resolver: if cli_options.no_npm() { + None + } else { + Some(NodeAndNpmReqResolver { + node_resolver: self.node_resolver().await?.clone(), + npm_req_resolver: self.npm_req_resolver().await?.clone(), + }) + }, + sloppy_imports_resolver: self.sloppy_imports_resolver()?.cloned(), + workspace_resolver: self.workspace_resolver().await?.clone(), + is_byonm: cli_options.use_byonm(), + maybe_vendor_dir: cli_options.vendor_dir_path(), + }))) + }) + .await + } + + pub async fn resolver(&self) -> Result<&Arc, AnyError> { self .services .resolver .get_or_try_init_async( async { let cli_options = self.cli_options()?; - Ok(Arc::new(CliGraphResolver::new(CliGraphResolverOptions { - sloppy_imports_resolver: self.sloppy_imports_resolver()?.cloned(), - node_resolver: Some(self.cli_node_resolver().await?.clone()), + Ok(Arc::new(CliResolver::new(CliResolverOptions { npm_resolver: if cli_options.no_npm() { None } else { Some(self.npm_resolver().await?.clone()) }, - workspace_resolver: self.workspace_resolver().await?.clone(), bare_node_builtins_enabled: cli_options .unstable_bare_node_builtins(), - maybe_jsx_import_source_config: cli_options - .workspace() - .to_maybe_jsx_import_source_config()?, - maybe_vendor_dir: cli_options.vendor_dir_path(), + deno_resolver: self.deno_resolver().await?.clone(), }))) } .boxed_local(), @@ -503,6 +597,7 @@ impl CliFactory { self.services.module_info_cache.get_or_try_init(|| { Ok(Arc::new(ModuleInfoCache::new( self.caches()?.dep_analysis_db(), + self.parsed_source_cache().clone(), ))) }) } @@ -531,6 +626,7 @@ impl CliFactory { ts_config_result.ts_config, )?; Ok(Arc::new(Emitter::new( + self.cjs_tracker()?.clone(), self.emit_cache()?.clone(), self.parsed_source_cache().clone(), transpile_options, @@ -554,7 +650,13 @@ impl CliFactory { async { Ok(Arc::new(NodeResolver::new( DenoFsNodeResolverEnv::new(self.fs().clone()), - self.npm_resolver().await?.clone().into_npm_resolver(), + self.in_npm_pkg_checker()?.clone(), + self + .npm_resolver() + .await? + .clone() + .into_npm_pkg_folder_resolver(), + self.pkg_json_resolver().clone(), ))) } .boxed_local(), @@ -572,23 +674,57 @@ impl CliFactory { let caches = self.caches()?; let node_analysis_cache = NodeAnalysisCache::new(caches.node_analysis_db()); - let node_resolver = self.cli_node_resolver().await?.clone(); + let node_resolver = self.node_resolver().await?.clone(); let cjs_esm_analyzer = CliCjsCodeAnalyzer::new( node_analysis_cache, + self.cjs_tracker()?.clone(), self.fs().clone(), - node_resolver, + Some(self.parsed_source_cache().clone()), ); Ok(Arc::new(NodeCodeTranslator::new( cjs_esm_analyzer, DenoFsNodeResolverEnv::new(self.fs().clone()), - self.node_resolver().await?.clone(), - self.npm_resolver().await?.clone().into_npm_resolver(), + self.in_npm_pkg_checker()?.clone(), + node_resolver, + self + .npm_resolver() + .await? + .clone() + .into_npm_pkg_folder_resolver(), + self.pkg_json_resolver().clone(), ))) }) .await } + pub async fn npm_req_resolver( + &self, + ) -> Result<&Arc, AnyError> { + self + .services + .npm_req_resolver + .get_or_try_init_async(async { + let npm_resolver = self.npm_resolver().await?; + Ok(Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions { + byonm_resolver: (npm_resolver.clone()).into_maybe_byonm(), + fs: CliDenoResolverFs(self.fs().clone()), + in_npm_pkg_checker: self.in_npm_pkg_checker()?.clone(), + node_resolver: self.node_resolver().await?.clone(), + npm_req_resolver: npm_resolver.clone().into_npm_req_resolver(), + }))) + }) + .await + } + + pub fn pkg_json_resolver(&self) -> &Arc { + self.services.pkg_json_resolver.get_or_init(|| { + Arc::new(PackageJsonResolver::new(DenoFsNodeResolverEnv::new( + self.fs().clone(), + ))) + }) + } + pub async fn type_checker(&self) -> Result<&Arc, AnyError> { self .services @@ -597,6 +733,10 @@ impl CliFactory { let cli_options = self.cli_options()?; Ok(Arc::new(TypeChecker::new( self.caches()?.clone(), + Arc::new(TypeCheckingCjsTracker::new( + self.cjs_tracker()?.clone(), + self.module_info_cache()?.clone(), + )), cli_options.clone(), self.module_graph_builder().await?.clone(), self.node_resolver().await?.clone(), @@ -615,17 +755,20 @@ impl CliFactory { .get_or_try_init_async(async { let cli_options = self.cli_options()?; Ok(Arc::new(ModuleGraphBuilder::new( - cli_options.clone(), self.caches()?.clone(), + self.cjs_tracker()?.clone(), + cli_options.clone(), + self.file_fetcher()?.clone(), self.fs().clone(), - self.resolver().await?.clone(), - self.npm_resolver().await?.clone(), - self.module_info_cache()?.clone(), - self.parsed_source_cache().clone(), + self.global_http_cache()?.clone(), + self.in_npm_pkg_checker()?.clone(), cli_options.maybe_lockfile().cloned(), self.maybe_file_watcher_reporter().clone(), - self.file_fetcher()?.clone(), - self.global_http_cache()?.clone(), + self.module_info_cache()?.clone(), + self.npm_resolver().await?.clone(), + self.parsed_source_cache().clone(), + self.resolver().await?.clone(), + self.root_permissions_container()?.clone(), ))) }) .await @@ -659,6 +802,7 @@ impl CliFactory { Ok(Arc::new(MainModuleGraphContainer::new( self.cli_options()?.clone(), self.module_load_preparer().await?.clone(), + self.root_permissions_container()?.clone(), ))) }) .await @@ -695,25 +839,21 @@ impl CliFactory { .await } - pub fn cjs_resolutions(&self) -> &Arc { - self.services.cjs_resolutions.get_or_init(Default::default) - } - - pub async fn cli_node_resolver( - &self, - ) -> Result<&Arc, AnyError> { - self - .services - .cli_node_resolver - .get_or_try_init_async(async { - Ok(Arc::new(CliNodeResolver::new( - self.cjs_resolutions().clone(), - self.fs().clone(), - self.node_resolver().await?.clone(), - self.npm_resolver().await?.clone(), - ))) - }) - .await + pub fn cjs_tracker(&self) -> Result<&Arc, AnyError> { + self.services.cjs_tracker.get_or_try_init(|| { + let options = self.cli_options()?; + Ok(Arc::new(CjsTracker::new( + self.in_npm_pkg_checker()?.clone(), + self.pkg_json_resolver().clone(), + if options.is_node_main() || options.unstable_detect_cjs() { + IsCjsResolutionMode::ImplicitTypeCommonJs + } else if options.detect_cjs() { + IsCjsResolutionMode::ExplicitTypeCommonJs + } else { + IsCjsResolutionMode::Disabled + }, + ))) + }) } pub fn permission_desc_parser( @@ -746,7 +886,10 @@ impl CliFactory { ) -> Result { let cli_options = self.cli_options()?; Ok(DenoCompileBinaryWriter::new( + self.cjs_tracker()?, + self.cli_options()?, self.deno_dir()?, + self.emitter()?, self.file_fetcher()?, self.http_client_provider(), self.npm_resolver().await?.as_ref(), @@ -755,30 +898,40 @@ impl CliFactory { )) } - pub fn create_permissions_container( + pub fn root_permissions_container( &self, - ) -> Result { - let desc_parser = self.permission_desc_parser()?.clone(); - let permissions = Permissions::from_options( - desc_parser.as_ref(), - &self.cli_options()?.permissions_options(), - )?; - Ok(PermissionsContainer::new(desc_parser, permissions)) + ) -> Result<&PermissionsContainer, AnyError> { + self + .services + .root_permissions_container + .get_or_try_init(|| { + let desc_parser = self.permission_desc_parser()?.clone(); + let permissions = Permissions::from_options( + desc_parser.as_ref(), + &self.cli_options()?.permissions_options(), + )?; + Ok(PermissionsContainer::new(desc_parser, permissions)) + }) } pub async fn create_cli_main_worker_factory( &self, ) -> Result { let cli_options = self.cli_options()?; + let fs = self.fs(); let node_resolver = self.node_resolver().await?; let npm_resolver = self.npm_resolver().await?; - let fs = self.fs(); - let cli_node_resolver = self.cli_node_resolver().await?; + let cli_npm_resolver = self.npm_resolver().await?.clone(); + let in_npm_pkg_checker = self.in_npm_pkg_checker()?; let maybe_file_watcher_communicator = if cli_options.has_hmr() { Some(self.watcher_communicator.clone().unwrap()) } else { None }; + let node_code_translator = self.node_code_translator().await?; + let cjs_tracker = self.cjs_tracker()?.clone(); + let pkg_json_resolver = self.pkg_json_resolver().clone(); + let npm_req_resolver = self.npm_req_resolver().await?; Ok(CliMainWorkerFactory::new( self.blob_store().clone(), @@ -788,37 +941,44 @@ impl CliFactory { None }, self.feature_checker()?.clone(), - self.fs().clone(), + fs.clone(), maybe_file_watcher_communicator, self.maybe_inspector_server()?.clone(), cli_options.maybe_lockfile().cloned(), Box::new(CliModuleLoaderFactory::new( cli_options, + cjs_tracker, if cli_options.code_cache_enabled() { Some(self.code_cache()?.clone()) } else { None }, self.emitter()?.clone(), + fs.clone(), + in_npm_pkg_checker.clone(), self.main_module_graph_container().await?.clone(), self.module_load_preparer().await?.clone(), - cli_node_resolver.clone(), + node_code_translator.clone(), + node_resolver.clone(), + npm_req_resolver.clone(), + cli_npm_resolver.clone(), NpmModuleLoader::new( - self.cjs_resolutions().clone(), - self.node_code_translator().await?.clone(), + self.cjs_tracker()?.clone(), fs.clone(), - cli_node_resolver.clone(), + node_code_translator.clone(), ), self.parsed_source_cache().clone(), self.resolver().await?.clone(), )), node_resolver.clone(), npm_resolver.clone(), - self.permission_desc_parser()?.clone(), + pkg_json_resolver, self.root_cert_store_provider().clone(), + self.root_permissions_container()?.clone(), StorageKeyResolver::from_options(cli_options), cli_options.sub_command().clone(), self.create_cli_main_worker_options()?, + self.cli_options()?.otel_config(), )) } @@ -867,7 +1027,6 @@ impl CliFactory { inspect_wait: cli_options.inspect_wait().is_some(), strace_ops: cli_options.strace_ops().clone(), is_inspecting: cli_options.is_inspecting(), - is_npm_main: cli_options.is_npm_main(), location: cli_options.location_flag().clone(), // if the user ran a binary command, we'll need to set process.argv[0] // to be the name of the binary command instead of deno diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 2f4b0b3dc5016f..640f83c35cbb98 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -21,8 +21,10 @@ use deno_core::url::Url; use deno_core::ModuleSpecifier; use deno_graph::source::LoaderChecksum; +use deno_path_util::url_to_file_path; use deno_runtime::deno_permissions::PermissionsContainer; use deno_runtime::deno_web::BlobStore; +use http::header; use log::debug; use std::borrow::Cow; use std::collections::HashMap; @@ -135,7 +137,7 @@ impl MemoryFiles { /// Fetch a source file from the local file system. fn fetch_local(specifier: &ModuleSpecifier) -> Result { - let local = specifier.to_file_path().map_err(|_| { + let local = url_to_file_path(specifier).map_err(|_| { uri_error(format!("Invalid file path.\n Specifier: {specifier}")) })?; // If it doesnt have a extension, we want to treat it as typescript by default @@ -162,8 +164,19 @@ fn get_validated_scheme( ) -> Result { let scheme = specifier.scheme(); if !SUPPORTED_SCHEMES.contains(&scheme) { + // NOTE(bartlomieju): this message list additional `npm` and `jsr` schemes, but they should actually be handled + // before `file_fetcher.rs` APIs are even hit. + let mut all_supported_schemes = SUPPORTED_SCHEMES.to_vec(); + all_supported_schemes.extend_from_slice(&["npm", "jsr"]); + all_supported_schemes.sort(); + let scheme_list = all_supported_schemes + .iter() + .map(|scheme| format!(" - \"{}\"", scheme)) + .collect::>() + .join("\n"); Err(generic_error(format!( - "Unsupported scheme \"{scheme}\" for module \"{specifier}\". Supported schemes: {SUPPORTED_SCHEMES:#?}" + "Unsupported scheme \"{scheme}\" for module \"{specifier}\". Supported schemes:\n{}", + scheme_list ))) } else { Ok(scheme.to_string()) @@ -173,35 +186,14 @@ fn get_validated_scheme( #[derive(Debug, Copy, Clone)] pub enum FetchPermissionsOptionRef<'a> { AllowAll, - Container(&'a PermissionsContainer), -} - -#[derive(Debug, Clone)] -pub enum FetchPermissionsOption { - AllowAll, - Container(PermissionsContainer), -} - -impl FetchPermissionsOption { - pub fn as_ref(&self) -> FetchPermissionsOptionRef { - match self { - FetchPermissionsOption::AllowAll => FetchPermissionsOptionRef::AllowAll, - FetchPermissionsOption::Container(container) => { - FetchPermissionsOptionRef::Container(container) - } - } - } -} - -impl From for FetchPermissionsOption { - fn from(value: PermissionsContainer) -> Self { - Self::Container(value) - } + DynamicContainer(&'a PermissionsContainer), + StaticContainer(&'a PermissionsContainer), } pub struct FetchOptions<'a> { pub specifier: &'a ModuleSpecifier, pub permissions: FetchPermissionsOptionRef<'a>, + pub maybe_auth: Option<(header::HeaderName, header::HeaderValue)>, pub maybe_accept: Option<&'a str>, pub maybe_cache_setting: Option<&'a CacheSetting>, } @@ -354,7 +346,7 @@ impl FileFetcher { ) })?; - let bytes = blob.read_all().await?; + let bytes = blob.read_all().await; let headers = HashMap::from([("content-type".to_string(), blob.media_type.clone())]); @@ -371,6 +363,7 @@ impl FileFetcher { maybe_accept: Option<&str>, cache_setting: &CacheSetting, maybe_checksum: Option<&LoaderChecksum>, + maybe_auth: Option<(header::HeaderName, header::HeaderValue)>, ) -> Result { debug!( "FileFetcher::fetch_remote_no_follow - specifier: {}", @@ -463,6 +456,7 @@ impl FileFetcher { .as_ref() .map(|(_, etag)| etag.clone()), maybe_auth_token: maybe_auth_token.clone(), + maybe_auth: maybe_auth.clone(), maybe_progress_guard: maybe_progress_guard.as_ref(), }) .await? @@ -559,12 +553,22 @@ impl FileFetcher { specifier: &ModuleSpecifier, ) -> Result { self - .fetch_inner(specifier, FetchPermissionsOptionRef::AllowAll) + .fetch_inner(specifier, None, FetchPermissionsOptionRef::AllowAll) + .await + } + + #[inline(always)] + pub async fn fetch_bypass_permissions_with_maybe_auth( + &self, + specifier: &ModuleSpecifier, + maybe_auth: Option<(header::HeaderName, header::HeaderValue)>, + ) -> Result { + self + .fetch_inner(specifier, maybe_auth, FetchPermissionsOptionRef::AllowAll) .await } /// Fetch a source file and asynchronously return it. - #[allow(dead_code)] // todo(25469): undo when merging #[inline(always)] pub async fn fetch( &self, @@ -572,19 +576,25 @@ impl FileFetcher { permissions: &PermissionsContainer, ) -> Result { self - .fetch_inner(specifier, FetchPermissionsOptionRef::Container(permissions)) + .fetch_inner( + specifier, + None, + FetchPermissionsOptionRef::StaticContainer(permissions), + ) .await } async fn fetch_inner( &self, specifier: &ModuleSpecifier, + maybe_auth: Option<(header::HeaderName, header::HeaderValue)>, permissions: FetchPermissionsOptionRef<'_>, ) -> Result { self .fetch_with_options(FetchOptions { specifier, permissions, + maybe_auth, maybe_accept: None, maybe_cache_setting: None, }) @@ -604,12 +614,14 @@ impl FileFetcher { max_redirect: usize, ) -> Result { let mut specifier = Cow::Borrowed(options.specifier); + let mut maybe_auth = options.maybe_auth.clone(); for _ in 0..=max_redirect { match self .fetch_no_follow_with_options(FetchNoFollowOptions { fetch_options: FetchOptions { specifier: &specifier, permissions: options.permissions, + maybe_auth: maybe_auth.clone(), maybe_accept: options.maybe_accept, maybe_cache_setting: options.maybe_cache_setting, }, @@ -621,6 +633,10 @@ impl FileFetcher { return Ok(file); } FileOrRedirect::Redirect(redirect_specifier) => { + // If we were redirected to another origin, don't send the auth header anymore. + if redirect_specifier.origin() != specifier.origin() { + maybe_auth = None; + } specifier = Cow::Owned(redirect_specifier); } } @@ -647,8 +663,17 @@ impl FileFetcher { FetchPermissionsOptionRef::AllowAll => { // allow } - FetchPermissionsOptionRef::Container(permissions) => { - permissions.check_specifier(specifier)?; + FetchPermissionsOptionRef::StaticContainer(permissions) => { + permissions.check_specifier( + specifier, + deno_runtime::deno_permissions::CheckSpecifierKind::Static, + )?; + } + FetchPermissionsOptionRef::DynamicContainer(permissions) => { + permissions.check_specifier( + specifier, + deno_runtime::deno_permissions::CheckSpecifierKind::Dynamic, + )?; } } if let Some(file) = self.memory_files.get(specifier) { @@ -676,6 +701,7 @@ impl FileFetcher { options.maybe_accept, options.maybe_cache_setting.unwrap_or(&self.cache_setting), maybe_checksum, + options.maybe_auth, ) .await } @@ -736,7 +762,7 @@ mod tests { maybe_temp_dir: Option, ) -> (FileFetcher, TempDir, Arc) { let temp_dir = maybe_temp_dir.unwrap_or_default(); - let location = temp_dir.path().join("deps").to_path_buf(); + let location = temp_dir.path().join("remote").to_path_buf(); let blob_store: Arc = Default::default(); let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv)), @@ -766,6 +792,7 @@ mod tests { FetchOptions { specifier, permissions: FetchPermissionsOptionRef::AllowAll, + maybe_auth: None, maybe_accept: None, maybe_cache_setting: Some(&file_fetcher.cache_setting), }, @@ -974,7 +1001,7 @@ mod tests { // This creates a totally new instance, simulating another Deno process // invocation and indicates to "cache bust". - let location = temp_dir.path().join("deps").to_path_buf(); + let location = temp_dir.path().join("remote").to_path_buf(); let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new( location, @@ -1000,7 +1027,7 @@ mod tests { async fn test_fetch_uses_cache() { let _http_server_guard = test_util::http_server(); let temp_dir = TempDir::new(); - let location = temp_dir.path().join("deps").to_path_buf(); + let location = temp_dir.path().join("remote").to_path_buf(); let specifier = resolve_url("http://localhost:4545/subdir/mismatch_ext.ts").unwrap(); @@ -1166,7 +1193,7 @@ mod tests { async fn test_fetch_uses_cache_with_redirects() { let _http_server_guard = test_util::http_server(); let temp_dir = TempDir::new(); - let location = temp_dir.path().join("deps").to_path_buf(); + let location = temp_dir.path().join("remote").to_path_buf(); let specifier = resolve_url("http://localhost:4548/subdir/mismatch_ext.ts").unwrap(); let redirected_specifier = @@ -1265,6 +1292,7 @@ mod tests { FetchOptions { specifier: &specifier, permissions: FetchPermissionsOptionRef::AllowAll, + maybe_auth: None, maybe_accept: None, maybe_cache_setting: Some(&file_fetcher.cache_setting), }, @@ -1278,6 +1306,7 @@ mod tests { FetchOptions { specifier: &specifier, permissions: FetchPermissionsOptionRef::AllowAll, + maybe_auth: None, maybe_accept: None, maybe_cache_setting: Some(&file_fetcher.cache_setting), }, @@ -1334,7 +1363,7 @@ mod tests { async fn test_fetch_no_remote() { let _http_server_guard = test_util::http_server(); let temp_dir = TempDir::new(); - let location = temp_dir.path().join("deps").to_path_buf(); + let location = temp_dir.path().join("remote").to_path_buf(); let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new( location, @@ -1360,7 +1389,7 @@ mod tests { async fn test_fetch_cache_only() { let _http_server_guard = test_util::http_server(); let temp_dir = TempDir::new(); - let location = temp_dir.path().join("deps").to_path_buf(); + let location = temp_dir.path().join("remote").to_path_buf(); let file_fetcher_01 = FileFetcher::new( Arc::new(GlobalHttpCache::new(location.clone(), RealDenoCacheEnv)), CacheSetting::Only, diff --git a/cli/graph_container.rs b/cli/graph_container.rs index 211b278e08166f..c463d71a6a44fe 100644 --- a/cli/graph_container.rs +++ b/cli/graph_container.rs @@ -9,9 +9,9 @@ use deno_core::error::AnyError; use deno_core::parking_lot::RwLock; use deno_graph::ModuleGraph; use deno_runtime::colors; +use deno_runtime::deno_permissions::PermissionsContainer; use crate::args::CliOptions; -use crate::file_fetcher::FetchPermissionsOption; use crate::module_loader::ModuleLoadPreparer; use crate::util::fs::collect_specifiers; use crate::util::path::is_script_ext; @@ -45,12 +45,14 @@ pub struct MainModuleGraphContainer { inner: Arc>>, cli_options: Arc, module_load_preparer: Arc, + root_permissions: PermissionsContainer, } impl MainModuleGraphContainer { pub fn new( cli_options: Arc, module_load_preparer: Arc, + root_permissions: PermissionsContainer, ) -> Self { Self { update_queue: Default::default(), @@ -59,6 +61,7 @@ impl MainModuleGraphContainer { )))), cli_options, module_load_preparer, + root_permissions, } } @@ -76,7 +79,7 @@ impl MainModuleGraphContainer { specifiers, false, self.cli_options.ts_type_lib_window(), - FetchPermissionsOption::AllowAll, + self.root_permissions.clone(), ext_overwrite, ) .await?; diff --git a/cli/graph_util.rs b/cli/graph_util.rs index 1add83eb990a0f..63997dc9ce8510 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -6,22 +6,28 @@ use crate::args::CliLockfile; use crate::args::CliOptions; use crate::args::DENO_DISABLE_PEDANTIC_NODE_WARNINGS; use crate::cache; +use crate::cache::FetchCacher; use crate::cache::GlobalHttpCache; use crate::cache::ModuleInfoCache; use crate::cache::ParsedSourceCache; use crate::colors; use crate::errors::get_error_class_name; -use crate::file_fetcher::FetchPermissionsOption; use crate::file_fetcher::FileFetcher; use crate::npm::CliNpmResolver; -use crate::resolver::CliGraphResolver; -use crate::resolver::SloppyImportsResolver; +use crate::resolver::CjsTracker; +use crate::resolver::CliResolver; +use crate::resolver::CliSloppyImportsResolver; +use crate::resolver::SloppyImportsCachedFs; use crate::tools::check; use crate::tools::check::TypeChecker; use crate::util::file_watcher::WatcherCommunicator; use crate::util::fs::canonicalize_path; +use deno_config::deno_json::JsxImportSourceConfig; use deno_config::workspace::JsrPackageConfig; +use deno_core::anyhow::bail; use deno_graph::source::LoaderChecksum; +use deno_graph::source::ResolutionKind; +use deno_graph::FillFromLockfileOptions; use deno_graph::JsrLoadError; use deno_graph::ModuleLoadError; use deno_graph::WorkspaceFastCheckOption; @@ -31,7 +37,6 @@ use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_core::ModuleSpecifier; use deno_graph::source::Loader; -use deno_graph::source::ResolutionMode; use deno_graph::source::ResolveError; use deno_graph::GraphKind; use deno_graph::ModuleError; @@ -39,27 +44,29 @@ use deno_graph::ModuleGraph; use deno_graph::ModuleGraphError; use deno_graph::ResolutionError; use deno_graph::SpecifierError; +use deno_path_util::url_to_file_path; +use deno_resolver::sloppy_imports::SloppyImportsResolutionKind; use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_node; -use deno_runtime::fs_util::specifier_to_file_path; +use deno_runtime::deno_permissions::PermissionsContainer; use deno_semver::jsr::JsrDepPackageReq; use deno_semver::package::PackageNv; -use deno_semver::Version; use import_map::ImportMapError; +use node_resolver::InNpmPackageChecker; use std::collections::HashSet; use std::error::Error; use std::ops::Deref; use std::path::PathBuf; use std::sync::Arc; -#[derive(Clone, Copy)] +#[derive(Clone)] pub struct GraphValidOptions { pub check_js: bool, pub kind: GraphKind, - pub is_vendoring: bool, - /// Whether to exit the process for lockfile errors. - /// Otherwise, surfaces lockfile errors as errors. - pub exit_lockfile_errors: bool, + /// Whether to exit the process for integrity check errors such as + /// lockfile checksum mismatches and JSR integrity failures. + /// Otherwise, surfaces integrity errors as errors. + pub exit_integrity_errors: bool, } /// Check if `roots` and their deps are available. Returns `Ok(())` if @@ -75,17 +82,54 @@ pub fn graph_valid( roots: &[ModuleSpecifier], options: GraphValidOptions, ) -> Result<(), AnyError> { - if options.exit_lockfile_errors { - graph_exit_lock_errors(graph); + if options.exit_integrity_errors { + graph_exit_integrity_errors(graph); } - let mut errors = graph + let mut errors = graph_walk_errors( + graph, + fs, + roots, + GraphWalkErrorsOptions { + check_js: options.check_js, + kind: options.kind, + }, + ); + if let Some(error) = errors.next() { + Err(error) + } else { + // finally surface the npm resolution result + if let Err(err) = &graph.npm_dep_graph_result { + return Err(custom_error( + get_error_class_name(err), + format_deno_graph_error(err.as_ref().deref()), + )); + } + Ok(()) + } +} + +#[derive(Clone)] +pub struct GraphWalkErrorsOptions { + pub check_js: bool, + pub kind: GraphKind, +} + +/// Walks the errors found in the module graph that should be surfaced to users +/// and enhances them with CLI information. +pub fn graph_walk_errors<'a>( + graph: &'a ModuleGraph, + fs: &'a Arc, + roots: &'a [ModuleSpecifier], + options: GraphWalkErrorsOptions, +) -> impl Iterator + 'a { + graph .walk( roots.iter(), deno_graph::WalkOptions { check_js: options.check_js, kind: options.kind, - follow_dynamic: options.is_vendoring, + follow_dynamic: false, prefer_fast_check_graph: false, }, ) @@ -109,7 +153,7 @@ pub fn graph_valid( ) } ModuleGraphError::ModuleError(error) => { - enhanced_lockfile_error_message(error) + enhanced_integrity_error_message(error) .or_else(|| enhanced_sloppy_imports_error_message(fs, error)) .unwrap_or_else(|| format_deno_graph_error(error)) } @@ -132,58 +176,20 @@ pub fn graph_valid( return None; } - if options.is_vendoring { - // warn about failing dynamic imports when vendoring, but don't fail completely - if matches!( - error, - ModuleGraphError::ModuleError(ModuleError::MissingDynamic(_, _)) - ) { - log::warn!("Ignoring: {}", message); - return None; - } - - // ignore invalid downgrades and invalid local imports when vendoring - match &error { - ModuleGraphError::ResolutionError(err) - | ModuleGraphError::TypesResolutionError(err) => { - if matches!( - err, - ResolutionError::InvalidDowngrade { .. } - | ResolutionError::InvalidLocalImport { .. } - ) { - return None; - } - } - ModuleGraphError::ModuleError(_) => {} - } - } - Some(custom_error(get_error_class_name(&error.into()), message)) - }); - if let Some(error) = errors.next() { - Err(error) - } else { - // finally surface the npm resolution result - if let Err(err) = &graph.npm_dep_graph_result { - return Err(custom_error( - get_error_class_name(err), - format_deno_graph_error(err.as_ref().deref()), - )); - } - Ok(()) - } + }) } -pub fn graph_exit_lock_errors(graph: &ModuleGraph) { +pub fn graph_exit_integrity_errors(graph: &ModuleGraph) { for error in graph.module_errors() { - exit_for_lockfile_error(error); + exit_for_integrity_error(error); } } -fn exit_for_lockfile_error(err: &ModuleError) { - if let Some(err_message) = enhanced_lockfile_error_message(err) { +fn exit_for_integrity_error(err: &ModuleError) { + if let Some(err_message) = enhanced_integrity_error_message(err) { log::error!("{} {}", colors::red("error:"), err_message); - std::process::exit(10); + deno_runtime::exit(10); } } @@ -249,22 +255,58 @@ impl ModuleGraphCreator { package_configs: &[JsrPackageConfig], build_fast_check_graph: bool, ) -> Result { + struct PublishLoader(FetchCacher); + impl Loader for PublishLoader { + fn load( + &self, + specifier: &deno_ast::ModuleSpecifier, + options: deno_graph::source::LoadOptions, + ) -> deno_graph::source::LoadFuture { + if specifier.scheme() == "bun" { + return Box::pin(std::future::ready(Ok(Some( + deno_graph::source::LoadResponse::External { + specifier: specifier.clone(), + }, + )))); + } + self.0.load(specifier, options) + } + } + fn graph_has_external_remote(graph: &ModuleGraph) -> bool { + // Earlier on, we marked external non-JSR modules as external. + // If the graph contains any of those, it would cause type checking + // to crash, so since publishing is going to fail anyway, skip type + // checking. + graph.modules().any(|module| match module { + deno_graph::Module::External(external_module) => { + matches!(external_module.specifier.scheme(), "http" | "https") + } + _ => false, + }) + } + let mut roots = Vec::new(); for package_config in package_configs { roots.extend(package_config.config_file.resolve_export_value_urls()?); } + + let loader = self.module_graph_builder.create_graph_loader(); + let mut publish_loader = PublishLoader(loader); let mut graph = self .create_graph_with_options(CreateGraphOptions { is_dynamic: false, graph_kind: deno_graph::GraphKind::All, roots, - loader: None, + loader: Some(&mut publish_loader), }) .await?; self.graph_valid(&graph)?; - if self.options.type_check_mode().is_true() { + if self.options.type_check_mode().is_true() + && !graph_has_external_remote(&graph) + { self.type_check_graph(graph.clone()).await?; } + if build_fast_check_graph { let fast_check_workspace_members = package_configs .iter() @@ -279,6 +321,7 @@ impl ModuleGraphCreator { }, )?; } + Ok(graph) } @@ -359,46 +402,55 @@ pub struct BuildFastCheckGraphOptions<'a> { } pub struct ModuleGraphBuilder { - options: Arc, caches: Arc, + cjs_tracker: Arc, + cli_options: Arc, + file_fetcher: Arc, fs: Arc, - resolver: Arc, - npm_resolver: Arc, - module_info_cache: Arc, - parsed_source_cache: Arc, + global_http_cache: Arc, + in_npm_pkg_checker: Arc, lockfile: Option>, maybe_file_watcher_reporter: Option, - file_fetcher: Arc, - global_http_cache: Arc, + module_info_cache: Arc, + npm_resolver: Arc, + parsed_source_cache: Arc, + resolver: Arc, + root_permissions_container: PermissionsContainer, } impl ModuleGraphBuilder { #[allow(clippy::too_many_arguments)] pub fn new( - options: Arc, caches: Arc, + cjs_tracker: Arc, + cli_options: Arc, + file_fetcher: Arc, fs: Arc, - resolver: Arc, - npm_resolver: Arc, - module_info_cache: Arc, - parsed_source_cache: Arc, + global_http_cache: Arc, + in_npm_pkg_checker: Arc, lockfile: Option>, maybe_file_watcher_reporter: Option, - file_fetcher: Arc, - global_http_cache: Arc, + module_info_cache: Arc, + npm_resolver: Arc, + parsed_source_cache: Arc, + resolver: Arc, + root_permissions_container: PermissionsContainer, ) -> Self { Self { - options, caches, + cjs_tracker, + cli_options, + file_fetcher, fs, - resolver, - npm_resolver, - module_info_cache, - parsed_source_cache, + global_http_cache, + in_npm_pkg_checker, lockfile, maybe_file_watcher_reporter, - file_fetcher, - global_http_cache, + module_info_cache, + npm_resolver, + parsed_source_cache, + resolver, + root_permissions_container, } } @@ -483,19 +535,17 @@ impl ModuleGraphBuilder { } let maybe_imports = if options.graph_kind.include_types() { - self.options.to_compiler_option_types()? + self.cli_options.to_compiler_option_types()? } else { Vec::new() }; - let analyzer = self - .module_info_cache - .as_module_analyzer(&self.parsed_source_cache); + let analyzer = self.module_info_cache.as_module_analyzer(); let mut loader = match options.loader { Some(loader) => MutLoaderRef::Borrowed(loader), None => MutLoaderRef::Owned(self.create_graph_loader()), }; let cli_resolver = &self.resolver; - let graph_resolver = cli_resolver.as_graph_resolver(); + let graph_resolver = self.create_graph_resolver()?; let graph_npm_resolver = cli_resolver.create_graph_npm_resolver(); let maybe_file_watcher_reporter = self .maybe_file_watcher_reporter @@ -520,7 +570,7 @@ impl ModuleGraphBuilder { npm_resolver: Some(&graph_npm_resolver), module_analyzer: &analyzer, reporter: maybe_file_watcher_reporter, - resolver: Some(graph_resolver), + resolver: Some(&graph_resolver), locker: locker.as_mut().map(|l| l as _), }, ) @@ -537,7 +587,7 @@ impl ModuleGraphBuilder { // ensure an "npm install" is done if the user has explicitly // opted into using a node_modules directory if self - .options + .cli_options .node_modules_dir()? .map(|m| m.uses_node_modules_dir()) .unwrap_or(false) @@ -553,33 +603,19 @@ impl ModuleGraphBuilder { // populate the information from the lockfile if let Some(lockfile) = &self.lockfile { let lockfile = lockfile.lock(); - for (from, to) in &lockfile.content.redirects { - if let Ok(from) = ModuleSpecifier::parse(from) { - if let Ok(to) = ModuleSpecifier::parse(to) { - if !matches!(from.scheme(), "file" | "npm" | "jsr") { - graph.redirects.insert(from, to); - } - } - } - } - for (req_dep, value) in &lockfile.content.packages.specifiers { - match req_dep.kind { - deno_semver::package::PackageKind::Jsr => { - if let Ok(version) = Version::parse_standard(value) { - graph.packages.add_nv( - req_dep.req.clone(), - PackageNv { - name: req_dep.req.name.clone(), - version, - }, - ); - } - } - deno_semver::package::PackageKind::Npm => { - // ignore - } - } - } + graph.fill_from_lockfile(FillFromLockfileOptions { + redirects: lockfile + .content + .redirects + .iter() + .map(|(from, to)| (from.as_str(), to.as_str())), + package_specifiers: lockfile + .content + .packages + .specifiers + .iter() + .map(|(dep, id)| (dep, id.as_str())), + }); } } @@ -587,6 +623,12 @@ impl ModuleGraphBuilder { let initial_package_deps_len = graph.packages.package_deps_sum(); let initial_package_mappings_len = graph.packages.mappings().len(); + if roots.iter().any(|r| r.scheme() == "npm") + && self.npm_resolver.as_byonm().is_some() + { + bail!("Resolving npm specifier entrypoints this way is currently not supported with \"nodeModules\": \"manual\". In the meantime, try with --node-modules-dir=auto instead"); + } + graph.build(roots, loader, options).await; let has_redirects_changed = graph.redirects.len() != initial_redirects_len; @@ -651,16 +693,16 @@ impl ModuleGraphBuilder { }; let parser = self.parsed_source_cache.as_capturing_parser(); let cli_resolver = &self.resolver; - let graph_resolver = cli_resolver.as_graph_resolver(); + let graph_resolver = self.create_graph_resolver()?; let graph_npm_resolver = cli_resolver.create_graph_npm_resolver(); graph.build_fast_check_type_graph( deno_graph::BuildFastCheckTypeGraphOptions { - jsr_url_provider: &CliJsrUrlProvider, + es_parser: Some(&parser), fast_check_cache: fast_check_cache.as_ref().map(|c| c as _), fast_check_dts: false, - module_parser: Some(&parser), - resolver: Some(graph_resolver), + jsr_url_provider: &CliJsrUrlProvider, + resolver: Some(&graph_resolver), npm_resolver: Some(&graph_npm_resolver), workspace_fast_check: options.workspace_fast_check, }, @@ -670,20 +712,27 @@ impl ModuleGraphBuilder { /// Creates the default loader used for creating a graph. pub fn create_graph_loader(&self) -> cache::FetchCacher { - self.create_fetch_cacher(FetchPermissionsOption::AllowAll) + self.create_fetch_cacher(self.root_permissions_container.clone()) } pub fn create_fetch_cacher( &self, - permissions: FetchPermissionsOption, + permissions: PermissionsContainer, ) -> cache::FetchCacher { cache::FetchCacher::new( self.file_fetcher.clone(), - self.options.resolve_file_header_overrides(), + self.fs.clone(), self.global_http_cache.clone(), - self.npm_resolver.clone(), + self.in_npm_pkg_checker.clone(), self.module_info_cache.clone(), - permissions, + cache::FetchCacherOptions { + file_header_overrides: self.cli_options.resolve_file_header_overrides(), + permissions, + is_deno_publish: matches!( + self.cli_options.sub_command(), + crate::args::DenoSubcommand::Publish { .. } + ), + }, ) } @@ -707,17 +756,28 @@ impl ModuleGraphBuilder { &self.fs, roots, GraphValidOptions { - is_vendoring: false, - kind: if self.options.type_check_mode().is_true() { + kind: if self.cli_options.type_check_mode().is_true() { GraphKind::All } else { GraphKind::CodeOnly }, - check_js: self.options.check_js(), - exit_lockfile_errors: true, + check_js: self.cli_options.check_js(), + exit_integrity_errors: true, }, ) } + + fn create_graph_resolver(&self) -> Result { + let jsx_import_source_config = self + .cli_options + .workspace() + .to_maybe_jsx_import_source_config()?; + Ok(CliGraphResolver { + cjs_tracker: &self.cjs_tracker, + resolver: &self.resolver, + jsx_import_source_config, + }) + } } /// Adds more explanatory information to a resolution error. @@ -755,8 +815,8 @@ fn enhanced_sloppy_imports_error_message( match error { ModuleError::LoadingErr(specifier, _, ModuleLoadError::Loader(_)) // ex. "Is a directory" error | ModuleError::Missing(specifier, _) => { - let additional_message = SloppyImportsResolver::new(fs.clone()) - .resolve(specifier, ResolutionMode::Execution)? + let additional_message = CliSloppyImportsResolver::new(SloppyImportsCachedFs::new(fs.clone())) + .resolve(specifier, SloppyImportsResolutionKind::Execution)? .as_suggestion_message(); Some(format!( "{} {} or run with --unstable-sloppy-imports", @@ -768,7 +828,7 @@ fn enhanced_sloppy_imports_error_message( } } -fn enhanced_lockfile_error_message(err: &ModuleError) -> Option { +fn enhanced_integrity_error_message(err: &ModuleError) -> Option { match err { ModuleError::LoadingErr( specifier, @@ -938,7 +998,7 @@ pub fn has_graph_root_local_dependent_changed( }, ); while let Some((s, _)) = dependent_specifiers.next() { - if let Ok(path) = specifier_to_file_path(s) { + if let Ok(path) = url_to_file_path(s) { if let Ok(path) = canonicalize_path(&path) { if canonicalized_changed_paths.contains(&path) { return true; @@ -980,7 +1040,11 @@ impl deno_graph::source::Reporter for FileWatcherReporter { ) { let mut file_paths = self.file_paths.lock(); if specifier.scheme() == "file" { - file_paths.push(specifier.to_file_path().unwrap()); + // Don't trust that the path is a valid path at this point: + // https://github.com/denoland/deno/issues/26209. + if let Ok(file_path) = specifier.to_file_path() { + file_paths.push(file_path); + } } if modules_done == modules_total { @@ -1057,12 +1121,12 @@ impl<'a> deno_graph::source::FileSystem for DenoGraphFsAdapter<'a> { } } -pub fn format_range_with_colors(range: &deno_graph::Range) -> String { +pub fn format_range_with_colors(referrer: &deno_graph::Range) -> String { format!( "{}:{}:{}", - colors::cyan(range.specifier.as_str()), - colors::yellow(&(range.start.line + 1).to_string()), - colors::yellow(&(range.start.character + 1).to_string()) + colors::cyan(referrer.specifier.as_str()), + colors::yellow(&(referrer.range.start.line + 1).to_string()), + colors::yellow(&(referrer.range.start.character + 1).to_string()) ) } @@ -1118,13 +1182,88 @@ fn format_deno_graph_error(err: &dyn Error) -> String { message } +#[derive(Debug)] +struct CliGraphResolver<'a> { + cjs_tracker: &'a CjsTracker, + resolver: &'a CliResolver, + jsx_import_source_config: Option, +} + +impl<'a> deno_graph::source::Resolver for CliGraphResolver<'a> { + fn default_jsx_import_source(&self) -> Option { + self + .jsx_import_source_config + .as_ref() + .and_then(|c| c.default_specifier.clone()) + } + + fn default_jsx_import_source_types(&self) -> Option { + self + .jsx_import_source_config + .as_ref() + .and_then(|c| c.default_types_specifier.clone()) + } + + fn jsx_import_source_module(&self) -> &str { + self + .jsx_import_source_config + .as_ref() + .map(|c| c.module.as_str()) + .unwrap_or(deno_graph::source::DEFAULT_JSX_IMPORT_SOURCE_MODULE) + } + + fn resolve( + &self, + raw_specifier: &str, + referrer_range: &deno_graph::Range, + resolution_kind: ResolutionKind, + ) -> Result { + self.resolver.resolve( + raw_specifier, + &referrer_range.specifier, + referrer_range.range.start, + referrer_range + .resolution_mode + .map(to_node_resolution_mode) + .unwrap_or_else(|| { + self + .cjs_tracker + .get_referrer_kind(&referrer_range.specifier) + }), + to_node_resolution_kind(resolution_kind), + ) + } +} + +pub fn to_node_resolution_kind( + kind: ResolutionKind, +) -> node_resolver::NodeResolutionKind { + match kind { + ResolutionKind::Execution => node_resolver::NodeResolutionKind::Execution, + ResolutionKind::Types => node_resolver::NodeResolutionKind::Types, + } +} + +pub fn to_node_resolution_mode( + mode: deno_graph::source::ResolutionMode, +) -> node_resolver::ResolutionMode { + match mode { + deno_graph::source::ResolutionMode::Import => { + node_resolver::ResolutionMode::Import + } + deno_graph::source::ResolutionMode::Require => { + node_resolver::ResolutionMode::Require + } + } +} + #[cfg(test)] mod test { use std::sync::Arc; use deno_ast::ModuleSpecifier; use deno_graph::source::ResolveError; - use deno_graph::Position; + use deno_graph::PositionRange; use deno_graph::Range; use deno_graph::ResolutionError; use deno_graph::SpecifierError; @@ -1145,8 +1284,8 @@ mod test { specifier: input.to_string(), range: Range { specifier, - start: Position::zeroed(), - end: Position::zeroed(), + resolution_mode: None, + range: PositionRange::zeroed(), }, }; assert_eq!(get_resolution_error_bare_node_specifier(&err), output); @@ -1161,8 +1300,8 @@ mod test { let err = ResolutionError::InvalidSpecifier { range: Range { specifier, - start: Position::zeroed(), - end: Position::zeroed(), + resolution_mode: None, + range: PositionRange::zeroed(), }, error: SpecifierError::ImportPrefixMissing { specifier: input.to_string(), diff --git a/cli/http_util.rs b/cli/http_util.rs index cf244c525a81b1..4b17936d688d9f 100644 --- a/cli/http_util.rs +++ b/cli/http_util.rs @@ -19,6 +19,7 @@ use deno_runtime::deno_fetch; use deno_runtime::deno_fetch::create_http_client; use deno_runtime::deno_fetch::CreateHttpClientOptions; use deno_runtime::deno_tls::RootCertStoreProvider; +use http::header; use http::header::HeaderName; use http::header::HeaderValue; use http::header::ACCEPT; @@ -204,6 +205,7 @@ pub struct FetchOnceArgs<'a> { pub maybe_accept: Option, pub maybe_etag: Option, pub maybe_auth_token: Option, + pub maybe_auth: Option<(header::HeaderName, header::HeaderValue)>, pub maybe_progress_guard: Option<&'a UpdateGuard>, } @@ -382,6 +384,8 @@ impl HttpClient { request .headers_mut() .insert(AUTHORIZATION, authorization_val); + } else if let Some((header, value)) = args.maybe_auth { + request.headers_mut().insert(header, value); } if let Some(accept) = args.maybe_accept { let accepts_val = HeaderValue::from_str(&accept)?; @@ -470,15 +474,23 @@ impl HttpClient { } } - pub async fn download_with_progress( + pub async fn download_with_progress_and_retries( &self, url: Url, maybe_header: Option<(HeaderName, HeaderValue)>, progress_guard: &UpdateGuard, ) -> Result>, DownloadError> { - self - .download_inner(url, maybe_header, Some(progress_guard)) - .await + crate::util::retry::retry( + || { + self.download_inner( + url.clone(), + maybe_header.clone(), + Some(progress_guard), + ) + }, + |e| matches!(e, DownloadError::BadResponse(_) | DownloadError::Fetch(_)), + ) + .await } pub async fn get_redirected_url( @@ -784,6 +796,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { @@ -810,6 +823,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { @@ -837,6 +851,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { @@ -858,6 +873,7 @@ mod test { maybe_etag: Some("33a64df551425fcc55e".to_string()), maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; assert_eq!(res.unwrap(), FetchOnceResult::NotModified); @@ -877,6 +893,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { @@ -906,6 +923,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Code(body, _)) = result { @@ -931,6 +949,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Redirect(url, _)) = result { @@ -966,6 +985,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { @@ -1013,6 +1033,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; @@ -1075,6 +1096,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; @@ -1128,6 +1150,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { @@ -1169,6 +1192,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { @@ -1191,6 +1215,7 @@ mod test { maybe_etag: Some("33a64df551425fcc55e".to_string()), maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; assert_eq!(res.unwrap(), FetchOnceResult::NotModified); @@ -1225,6 +1250,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { @@ -1254,6 +1280,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; assert!(result.is_err()); @@ -1275,6 +1302,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; @@ -1298,6 +1326,7 @@ mod test { maybe_etag: None, maybe_auth_token: None, maybe_progress_guard: None, + maybe_auth: None, }) .await; diff --git a/cli/js/40_jupyter.js b/cli/js/40_jupyter.js index ace50d6dccbc0e..198b6a350254eb 100644 --- a/cli/js/40_jupyter.js +++ b/cli/js/40_jupyter.js @@ -177,6 +177,52 @@ function isCanvasLike(obj) { return obj !== null && typeof obj === "object" && "toDataURL" in obj; } +function isJpg(obj) { + // Check if obj is a Uint8Array + if (!(obj instanceof Uint8Array)) { + return false; + } + + // JPG files start with the magic bytes FF D8 + if (obj.length < 2 || obj[0] !== 0xFF || obj[1] !== 0xD8) { + return false; + } + + // JPG files end with the magic bytes FF D9 + if ( + obj.length < 2 || obj[obj.length - 2] !== 0xFF || + obj[obj.length - 1] !== 0xD9 + ) { + return false; + } + + return true; +} + +function isPng(obj) { + // Check if obj is a Uint8Array + if (!(obj instanceof Uint8Array)) { + return false; + } + + // PNG files start with a specific 8-byte signature + const pngSignature = [137, 80, 78, 71, 13, 10, 26, 10]; + + // Check if the array is at least as long as the signature + if (obj.length < pngSignature.length) { + return false; + } + + // Check each byte of the signature + for (let i = 0; i < pngSignature.length; i++) { + if (obj[i] !== pngSignature[i]) { + return false; + } + } + + return true; +} + /** Possible HTML and SVG Elements */ function isSVGElementLike(obj) { return obj !== null && typeof obj === "object" && "outerHTML" in obj && @@ -233,6 +279,16 @@ async function format(obj) { if (isDataFrameLike(obj)) { return extractDataFrame(obj); } + if (isJpg(obj)) { + return { + "image/jpeg": core.ops.op_base64_encode(obj), + }; + } + if (isPng(obj)) { + return { + "image/png": core.ops.op_base64_encode(obj), + }; + } if (isSVGElementLike(obj)) { return { "image/svg+xml": obj.outerHTML, @@ -314,6 +370,28 @@ const html = createTaggedTemplateDisplayable("text/html"); */ const svg = createTaggedTemplateDisplayable("image/svg+xml"); +function image(obj) { + if (typeof obj === "string") { + try { + obj = Deno.readFileSync(obj); + } catch { + // pass + } + } + + if (isJpg(obj)) { + return makeDisplayable({ "image/jpeg": core.ops.op_base64_encode(obj) }); + } + + if (isPng(obj)) { + return makeDisplayable({ "image/png": core.ops.op_base64_encode(obj) }); + } + + throw new TypeError( + "Object is not a valid image or a path to an image. `Deno.jupyter.image` supports displaying JPG or PNG images.", + ); +} + function isMediaBundle(obj) { if (obj == null || typeof obj !== "object" || Array.isArray(obj)) { return false; @@ -465,6 +543,7 @@ function enableJupyter() { md, html, svg, + image, $display, }; } diff --git a/cli/lsp/analysis.rs b/cli/lsp/analysis.rs index fee9c86cbe7ffe..65ce330dfce404 100644 --- a/cli/lsp/analysis.rs +++ b/cli/lsp/analysis.rs @@ -2,6 +2,7 @@ use super::diagnostics::DenoDiagnostic; use super::diagnostics::DiagnosticSource; +use super::documents::Document; use super::documents::Documents; use super::language_server; use super::resolver::LspResolver; @@ -9,13 +10,16 @@ use super::tsc; use super::urls::url_to_uri; use crate::args::jsr_url; +use crate::lsp::logging::lsp_warn; +use crate::lsp::search::PackageSearchApi; use crate::tools::lint::CliLinter; +use crate::util::path::relative_specifier; +use deno_config::workspace::MappedResolution; use deno_lint::diagnostic::LintDiagnosticRange; use deno_ast::SourceRange; use deno_ast::SourceRangedForSpanned; use deno_ast::SourceTextInfo; -use deno_core::anyhow::anyhow; use deno_core::error::custom_error; use deno_core::error::AnyError; use deno_core::serde::Deserialize; @@ -23,8 +27,8 @@ use deno_core::serde::Serialize; use deno_core::serde_json; use deno_core::serde_json::json; use deno_core::ModuleSpecifier; +use deno_path_util::url_to_file_path; use deno_runtime::deno_node::PathClean; -use deno_runtime::fs_util::specifier_to_file_path; use deno_semver::jsr::JsrPackageNvReference; use deno_semver::jsr::JsrPackageReqReference; use deno_semver::npm::NpmPackageReqReference; @@ -34,13 +38,16 @@ use deno_semver::package::PackageReq; use deno_semver::package::PackageReqReference; use deno_semver::Version; use import_map::ImportMap; -use node_resolver::NpmResolver; +use node_resolver::NodeResolutionKind; +use node_resolver::ResolutionMode; use once_cell::sync::Lazy; use regex::Regex; +use std::borrow::Cow; use std::cmp::Ordering; use std::collections::HashMap; use std::collections::HashSet; use std::path::Path; +use text_lines::LineAndColumnIndex; use tower_lsp::lsp_types as lsp; use tower_lsp::lsp_types::Position; use tower_lsp::lsp_types::Range; @@ -225,6 +232,8 @@ pub struct TsResponseImportMapper<'a> { documents: &'a Documents, maybe_import_map: Option<&'a ImportMap>, resolver: &'a LspResolver, + tsc_specifier_map: &'a tsc::TscSpecifierMap, + file_referrer: ModuleSpecifier, } impl<'a> TsResponseImportMapper<'a> { @@ -232,11 +241,15 @@ impl<'a> TsResponseImportMapper<'a> { documents: &'a Documents, maybe_import_map: Option<&'a ImportMap>, resolver: &'a LspResolver, + tsc_specifier_map: &'a tsc::TscSpecifierMap, + file_referrer: &ModuleSpecifier, ) -> Self { Self { documents, maybe_import_map, resolver, + tsc_specifier_map, + file_referrer: file_referrer.clone(), } } @@ -257,8 +270,6 @@ impl<'a> TsResponseImportMapper<'a> { } } - let file_referrer = self.documents.get_file_referrer(referrer); - if let Some(jsr_path) = specifier.as_str().strip_prefix(jsr_url().as_str()) { let mut segments = jsr_path.split('/'); @@ -273,7 +284,7 @@ impl<'a> TsResponseImportMapper<'a> { let export = self.resolver.jsr_lookup_export_for_path( &nv, &path, - file_referrer.as_deref(), + Some(&self.file_referrer), )?; let sub_path = (export != ".").then_some(export); let mut req = None; @@ -299,7 +310,7 @@ impl<'a> TsResponseImportMapper<'a> { req = req.or_else(|| { self .resolver - .jsr_lookup_req_for_nv(&nv, file_referrer.as_deref()) + .jsr_lookup_req_for_nv(&nv, Some(&self.file_referrer)) }); let spec_str = if let Some(req) = req { let req_ref = PackageReqReference { req, sub_path }; @@ -329,9 +340,13 @@ impl<'a> TsResponseImportMapper<'a> { if let Some(npm_resolver) = self .resolver - .maybe_managed_npm_resolver(file_referrer.as_deref()) + .maybe_managed_npm_resolver(Some(&self.file_referrer)) { - if npm_resolver.in_npm_package(specifier) { + let in_npm_pkg = self + .resolver + .in_npm_pkg_checker(Some(&self.file_referrer)) + .in_npm_package(specifier); + if in_npm_pkg { if let Ok(Some(pkg_id)) = npm_resolver.resolve_pkg_id_from_specifier(specifier) { @@ -378,6 +393,11 @@ impl<'a> TsResponseImportMapper<'a> { } } } + } else if let Some(dep_name) = self + .resolver + .file_url_to_package_json_dep(specifier, Some(&self.file_referrer)) + { + return Some(dep_name); } // check if the import map has this specifier @@ -401,7 +421,7 @@ impl<'a> TsResponseImportMapper<'a> { .flatten()?; let root_folder = package_json.path.parent()?; - let specifier_path = specifier_to_file_path(specifier).ok()?; + let specifier_path = url_to_file_path(specifier).ok()?; let mut search_paths = vec![specifier_path.clone()]; // TypeScript will provide a .js extension for quick fixes, so do // a search for the .d.ts file instead @@ -447,24 +467,59 @@ impl<'a> TsResponseImportMapper<'a> { &self, specifier: &str, referrer: &ModuleSpecifier, + resolution_mode: ResolutionMode, ) -> Option { - if let Ok(specifier) = referrer.join(specifier) { - if let Some(specifier) = self.check_specifier(&specifier, referrer) { - return Some(specifier); - } - } - let specifier = specifier.strip_suffix(".js").unwrap_or(specifier); - for ext in SUPPORTED_EXTENSIONS { - let specifier_with_ext = format!("{specifier}{ext}"); - if self - .documents - .contains_import(&specifier_with_ext, referrer) + let specifier_stem = specifier.strip_suffix(".js").unwrap_or(specifier); + let specifiers = std::iter::once(Cow::Borrowed(specifier)).chain( + SUPPORTED_EXTENSIONS + .iter() + .map(|ext| Cow::Owned(format!("{specifier_stem}{ext}"))), + ); + for specifier in specifiers { + if let Some(specifier) = self + .resolver + .as_cli_resolver(Some(&self.file_referrer)) + .resolve( + &specifier, + referrer, + deno_graph::Position::zeroed(), + resolution_mode, + NodeResolutionKind::Types, + ) + .ok() + .and_then(|s| self.tsc_specifier_map.normalize(s.as_str()).ok()) + .filter(|s| self.documents.exists(s, Some(&self.file_referrer))) { - return Some(specifier_with_ext); + if let Some(specifier) = self + .check_specifier(&specifier, referrer) + .or_else(|| relative_specifier(referrer, &specifier)) + .filter(|s| !s.contains("/node_modules/")) + { + return Some(specifier); + } } } None } + + pub fn is_valid_import( + &self, + specifier_text: &str, + referrer: &ModuleSpecifier, + resolution_mode: ResolutionMode, + ) -> bool { + self + .resolver + .as_cli_resolver(Some(&self.file_referrer)) + .resolve( + specifier_text, + referrer, + deno_graph::Position::zeroed(), + resolution_mode, + NodeResolutionKind::Types, + ) + .is_ok() + } } fn try_reverse_map_package_json_exports( @@ -529,9 +584,11 @@ fn try_reverse_map_package_json_exports( /// like an import and rewrite the import specifier to include the extension pub fn fix_ts_import_changes( referrer: &ModuleSpecifier, + resolution_mode: ResolutionMode, changes: &[tsc::FileTextChanges], - import_mapper: &TsResponseImportMapper, + language_server: &language_server::Inner, ) -> Result, AnyError> { + let import_mapper = language_server.get_ts_response_import_mapper(referrer); let mut r = Vec::new(); for change in changes { let mut text_changes = Vec::new(); @@ -544,8 +601,8 @@ pub fn fix_ts_import_changes( if let Some(captures) = IMPORT_SPECIFIER_RE.captures(line) { let specifier = captures.iter().skip(1).find_map(|s| s).unwrap().as_str(); - if let Some(new_specifier) = - import_mapper.check_unresolved_specifier(specifier, referrer) + if let Some(new_specifier) = import_mapper + .check_unresolved_specifier(specifier, referrer, resolution_mode) { line.replace(specifier, &new_specifier) } else { @@ -573,66 +630,67 @@ pub fn fix_ts_import_changes( /// Fix tsc import code actions so that the module specifier is correct for /// resolution by Deno (includes the extension). -fn fix_ts_import_action( +fn fix_ts_import_action<'a>( referrer: &ModuleSpecifier, - action: &tsc::CodeFixAction, - import_mapper: &TsResponseImportMapper, -) -> Result { - if matches!( + resolution_mode: ResolutionMode, + action: &'a tsc::CodeFixAction, + language_server: &language_server::Inner, +) -> Option> { + if !matches!( action.fix_name.as_str(), "import" | "fixMissingFunctionDeclaration" ) { - let change = action + return Some(Cow::Borrowed(action)); + } + let specifier = (|| { + let text_change = action.changes.first()?.text_changes.first()?; + let captures = IMPORT_SPECIFIER_RE.captures(&text_change.new_text)?; + Some(captures.get(1)?.as_str()) + })(); + let Some(specifier) = specifier else { + return Some(Cow::Borrowed(action)); + }; + let import_mapper = language_server.get_ts_response_import_mapper(referrer); + if let Some(new_specifier) = import_mapper.check_unresolved_specifier( + specifier, + referrer, + resolution_mode, + ) { + let description = action.description.replace(specifier, &new_specifier); + let changes = action .changes - .first() - .ok_or_else(|| anyhow!("Unexpected action changes."))?; - let text_change = change - .text_changes - .first() - .ok_or_else(|| anyhow!("Missing text change."))?; - if let Some(captures) = IMPORT_SPECIFIER_RE.captures(&text_change.new_text) - { - let specifier = captures - .get(1) - .ok_or_else(|| anyhow!("Missing capture."))? - .as_str(); - if let Some(new_specifier) = - import_mapper.check_unresolved_specifier(specifier, referrer) - { - let description = action.description.replace(specifier, &new_specifier); - let changes = action - .changes + .iter() + .map(|c| { + let text_changes = c + .text_changes .iter() - .map(|c| { - let text_changes = c - .text_changes - .iter() - .map(|tc| tsc::TextChange { - span: tc.span.clone(), - new_text: tc.new_text.replace(specifier, &new_specifier), - }) - .collect(); - tsc::FileTextChanges { - file_name: c.file_name.clone(), - text_changes, - is_new_file: c.is_new_file, - } + .map(|tc| tsc::TextChange { + span: tc.span.clone(), + new_text: tc.new_text.replace(specifier, &new_specifier), }) .collect(); - - return Ok(tsc::CodeFixAction { - description, - changes, - commands: None, - fix_name: action.fix_name.clone(), - fix_id: None, - fix_all_description: None, - }); - } - } + tsc::FileTextChanges { + file_name: c.file_name.clone(), + text_changes, + is_new_file: c.is_new_file, + } + }) + .collect(); + + Some(Cow::Owned(tsc::CodeFixAction { + description, + changes, + commands: None, + fix_name: action.fix_name.clone(), + fix_id: None, + fix_all_description: None, + })) + } else if !import_mapper.is_valid_import(specifier, referrer, resolution_mode) + { + None + } else { + Some(Cow::Borrowed(action)) } - - Ok(action.clone()) } /// Determines if two TypeScript diagnostic codes are effectively equivalent. @@ -693,8 +751,14 @@ pub fn ts_changes_to_edit( ) -> Result, AnyError> { let mut text_document_edits = Vec::new(); for change in changes { - let text_document_edit = change.to_text_document_edit(language_server)?; - text_document_edits.push(text_document_edit); + let edit = match change.to_text_document_edit(language_server) { + Ok(e) => e, + Err(err) => { + lsp_warn!("Couldn't covert text document edit: {:#}", err); + continue; + } + }; + text_document_edits.push(edit); } Ok(Some(lsp::WorkspaceEdit { changes: None, @@ -703,7 +767,7 @@ pub fn ts_changes_to_edit( })) } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct CodeActionData { pub specifier: ModuleSpecifier, @@ -956,6 +1020,7 @@ impl CodeActionCollection { pub fn add_ts_fix_action( &mut self, specifier: &ModuleSpecifier, + resolution_mode: ResolutionMode, action: &tsc::CodeFixAction, diagnostic: &lsp::Diagnostic, language_server: &language_server::Inner, @@ -973,11 +1038,11 @@ impl CodeActionCollection { "The action returned from TypeScript is unsupported.", )); } - let action = fix_ts_import_action( - specifier, - action, - &language_server.get_ts_response_import_mapper(specifier), - )?; + let Some(action) = + fix_ts_import_action(specifier, resolution_mode, action, language_server) + else { + return Ok(()); + }; let edit = ts_changes_to_edit(&action.changes, language_server)?; let code_action = lsp::CodeAction { title: action.description.clone(), @@ -997,7 +1062,7 @@ impl CodeActionCollection { }); self .actions - .push(CodeActionKind::Tsc(code_action, action.clone())); + .push(CodeActionKind::Tsc(code_action, action.as_ref().clone())); if let Some(fix_id) = &action.fix_id { if let Some(CodeActionKind::Tsc(existing_fix_all, existing_action)) = @@ -1024,10 +1089,12 @@ impl CodeActionCollection { specifier: &ModuleSpecifier, diagnostic: &lsp::Diagnostic, ) { - let data = Some(json!({ - "specifier": specifier, - "fixId": action.fix_id, - })); + let data = action.fix_id.as_ref().map(|fix_id| { + json!(CodeActionData { + specifier: specifier.clone(), + fix_id: fix_id.clone(), + }) + }); let title = if let Some(description) = &action.fix_all_description { description.clone() } else { @@ -1151,6 +1218,190 @@ impl CodeActionCollection { ..Default::default() })); } + + pub async fn add_source_actions( + &mut self, + document: &Document, + range: &lsp::Range, + language_server: &language_server::Inner, + ) { + fn import_start_from_specifier( + document: &Document, + import: &deno_graph::Import, + ) -> Option { + // find the top level statement that contains the specifier + let parsed_source = document.maybe_parsed_source()?.as_ref().ok()?; + let text_info = parsed_source.text_info_lazy(); + let specifier_range = SourceRange::new( + text_info.loc_to_source_pos(LineAndColumnIndex { + line_index: import.specifier_range.range.start.line, + column_index: import.specifier_range.range.start.character, + }), + text_info.loc_to_source_pos(LineAndColumnIndex { + line_index: import.specifier_range.range.end.line, + column_index: import.specifier_range.range.end.character, + }), + ); + + parsed_source + .program_ref() + .body() + .find(|i| i.range().contains(&specifier_range)) + .map(|i| text_info.line_and_column_index(i.range().start)) + } + + async fn deno_types_for_npm_action( + document: &Document, + range: &lsp::Range, + language_server: &language_server::Inner, + ) -> Option { + let (dep_key, dependency, _) = + document.get_maybe_dependency(&range.end)?; + if dependency.maybe_deno_types_specifier.is_some() { + return None; + } + if dependency.maybe_code.maybe_specifier().is_none() + && dependency.maybe_type.maybe_specifier().is_none() + { + // We're using byonm and the package is not cached. + return None; + } + let position = deno_graph::Position::new( + range.end.line as usize, + range.end.character as usize, + ); + let import_start = dependency.imports.iter().find_map(|i| { + if json!(i.kind) != json!("es") && json!(i.kind) != json!("tsType") { + return None; + } + if !i.specifier_range.includes(position) { + return None; + } + + import_start_from_specifier(document, i) + })?; + let referrer = document.specifier(); + let resolution_mode = document.resolution_mode(); + let file_referrer = document.file_referrer(); + let config_data = language_server + .config + .tree + .data_for_specifier(file_referrer?)?; + let workspace_resolver = config_data.resolver.clone(); + let npm_ref = if let Ok(resolution) = + workspace_resolver.resolve(&dep_key, document.specifier()) + { + let specifier = match resolution { + MappedResolution::Normal { specifier, .. } + | MappedResolution::ImportMap { specifier, .. } => specifier, + _ => { + return None; + } + }; + NpmPackageReqReference::from_specifier(&specifier).ok()? + } else { + // Only resolve bare package.json deps for byonm. + if !config_data.byonm { + return None; + } + if !language_server.resolver.is_bare_package_json_dep( + &dep_key, + referrer, + resolution_mode, + ) { + return None; + } + NpmPackageReqReference::from_str(&format!("npm:{}", &dep_key)).ok()? + }; + let package_name = &npm_ref.req().name; + if package_name.starts_with("@types/") { + return None; + } + let managed_npm_resolver = language_server + .resolver + .maybe_managed_npm_resolver(file_referrer); + if let Some(npm_resolver) = managed_npm_resolver { + if !npm_resolver.is_pkg_req_folder_cached(npm_ref.req()) { + return None; + } + } + if language_server + .resolver + .npm_to_file_url(&npm_ref, referrer, resolution_mode, file_referrer) + .is_some() + { + // The package import has types. + return None; + } + let types_package_name = format!("@types/{package_name}"); + let types_package_version = language_server + .npm_search_api + .versions(&types_package_name) + .await + .ok() + .and_then(|versions| versions.first().cloned())?; + let types_specifier_text = + if let Some(npm_resolver) = managed_npm_resolver { + let mut specifier_text = if let Some(req) = + npm_resolver.top_package_req_for_name(&types_package_name) + { + format!("npm:{req}") + } else { + format!("npm:{}@^{}", &types_package_name, types_package_version) + }; + let specifier = ModuleSpecifier::parse(&specifier_text).ok()?; + if let Some(file_referrer) = file_referrer { + if let Some(text) = language_server + .get_ts_response_import_mapper(file_referrer) + .check_specifier(&specifier, referrer) + { + specifier_text = text; + } + } + specifier_text + } else { + types_package_name.clone() + }; + let uri = language_server + .url_map + .specifier_to_uri(referrer, file_referrer) + .ok()?; + let position = lsp::Position { + line: import_start.line_index as u32, + character: import_start.column_index as u32, + }; + let new_text = format!( + "{}// @deno-types=\"{}\"\n", + if position.character == 0 { "" } else { "\n" }, + &types_specifier_text + ); + let text_edit = lsp::TextEdit { + range: lsp::Range { + start: position, + end: position, + }, + new_text, + }; + Some(lsp::CodeAction { + title: format!( + "Add @deno-types directive for \"{}\"", + &types_specifier_text + ), + kind: Some(lsp::CodeActionKind::QUICKFIX), + diagnostics: None, + edit: Some(lsp::WorkspaceEdit { + changes: Some([(uri, vec![text_edit])].into_iter().collect()), + ..Default::default() + }), + ..Default::default() + }) + } + if let Some(action) = + deno_types_for_npm_action(document, range, language_server).await + { + self.actions.push(CodeActionKind::Deno(action)); + } + } } /// Prepend the whitespace characters found at the start of line_content to content. diff --git a/cli/lsp/cache.rs b/cli/lsp/cache.rs index a32087842d5e04..fbf9ea6f1b12f9 100644 --- a/cli/lsp/cache.rs +++ b/cli/lsp/cache.rs @@ -10,7 +10,7 @@ use crate::lsp::logging::lsp_warn; use deno_core::url::Url; use deno_core::ModuleSpecifier; -use deno_runtime::fs_util::specifier_to_file_path; +use deno_path_util::url_to_file_path; use std::collections::BTreeMap; use std::fs; use std::path::Path; @@ -24,7 +24,7 @@ pub fn calculate_fs_version( ) -> Option { match specifier.scheme() { "npm" | "node" | "data" | "blob" => None, - "file" => specifier_to_file_path(specifier) + "file" => url_to_file_path(specifier) .ok() .and_then(|path| calculate_fs_version_at_path(&path)), _ => calculate_fs_version_in_cache(cache, specifier, file_referrer), @@ -82,7 +82,7 @@ impl Default for LspCache { impl LspCache { pub fn new(global_cache_url: Option) -> Self { let global_cache_path = global_cache_url.and_then(|s| { - specifier_to_file_path(&s) + url_to_file_path(&s) .inspect(|p| { lsp_log!("Resolved global cache path: \"{}\"", p.to_string_lossy()); }) @@ -94,7 +94,7 @@ impl LspCache { let deno_dir = DenoDir::new(global_cache_path) .expect("should be infallible with absolute custom root"); let global = Arc::new(GlobalHttpCache::new( - deno_dir.deps_folder_path(), + deno_dir.remote_folder_path(), crate::cache::RealDenoCacheEnv, )); Self { @@ -165,7 +165,7 @@ impl LspCache { &self, specifier: &ModuleSpecifier, ) -> Option { - let path = specifier_to_file_path(specifier).ok()?; + let path = url_to_file_path(specifier).ok()?; let vendor = self .vendors_by_scope .iter() @@ -176,7 +176,7 @@ impl LspCache { } pub fn is_valid_file_referrer(&self, specifier: &ModuleSpecifier) -> bool { - if let Ok(path) = specifier_to_file_path(specifier) { + if let Ok(path) = url_to_file_path(specifier) { if !path.starts_with(&self.deno_dir().root) { return true; } diff --git a/cli/lsp/capabilities.rs b/cli/lsp/capabilities.rs index e93d3b7c20e5b5..5cdb1224d85780 100644 --- a/cli/lsp/capabilities.rs +++ b/cli/lsp/capabilities.rs @@ -147,11 +147,11 @@ pub fn server_capabilities( moniker_provider: None, experimental: Some(json!({ "denoConfigTasks": true, - "testingApi":true, + "testingApi": true, + "didRefreshDenoConfigurationTreeNotifications": true, })), inlay_hint_provider: Some(OneOf::Left(true)), position_encoding: None, - // TODO(nayeemrmn): Support pull-based diagnostics. diagnostic_provider: None, inline_value_provider: None, inline_completion_provider: None, diff --git a/cli/lsp/client.rs b/cli/lsp/client.rs index b3f0d64fa69d33..65865d5b324bdc 100644 --- a/cli/lsp/client.rs +++ b/cli/lsp/client.rs @@ -92,6 +92,19 @@ impl Client { }); } + pub fn send_did_refresh_deno_configuration_tree_notification( + &self, + params: lsp_custom::DidRefreshDenoConfigurationTreeNotificationParams, + ) { + // do on a task in case the caller currently is in the lsp lock + let client = self.0.clone(); + spawn(async move { + client + .send_did_refresh_deno_configuration_tree_notification(params) + .await; + }); + } + pub fn send_did_change_deno_configuration_notification( &self, params: lsp_custom::DidChangeDenoConfigurationNotificationParams, @@ -169,6 +182,10 @@ trait ClientTrait: Send + Sync { params: lsp_custom::DiagnosticBatchNotificationParams, ); async fn send_test_notification(&self, params: TestingNotification); + async fn send_did_refresh_deno_configuration_tree_notification( + &self, + params: lsp_custom::DidRefreshDenoConfigurationTreeNotificationParams, + ); async fn send_did_change_deno_configuration_notification( &self, params: lsp_custom::DidChangeDenoConfigurationNotificationParams, @@ -249,6 +266,18 @@ impl ClientTrait for TowerClient { } } + async fn send_did_refresh_deno_configuration_tree_notification( + &self, + params: lsp_custom::DidRefreshDenoConfigurationTreeNotificationParams, + ) { + self + .0 + .send_notification::( + params, + ) + .await + } + async fn send_did_change_deno_configuration_notification( &self, params: lsp_custom::DidChangeDenoConfigurationNotificationParams, @@ -366,6 +395,12 @@ impl ClientTrait for ReplClient { async fn send_test_notification(&self, _params: TestingNotification) {} + async fn send_did_refresh_deno_configuration_tree_notification( + &self, + _params: lsp_custom::DidRefreshDenoConfigurationTreeNotificationParams, + ) { + } + async fn send_did_change_deno_configuration_notification( &self, _params: lsp_custom::DidChangeDenoConfigurationNotificationParams, diff --git a/cli/lsp/code_lens.rs b/cli/lsp/code_lens.rs index e117888fba66f4..a57ca3ac9fe141 100644 --- a/cli/lsp/code_lens.rs +++ b/cli/lsp/code_lens.rs @@ -421,7 +421,7 @@ pub fn collect_test( ) -> Result, AnyError> { let mut collector = DenoTestCollector::new(specifier.clone(), parsed_source.clone()); - parsed_source.module().visit_with(&mut collector); + parsed_source.program().visit_with(&mut collector); Ok(collector.take()) } @@ -581,7 +581,7 @@ mod tests { .unwrap(); let mut collector = DenoTestCollector::new(specifier, parsed_module.clone()); - parsed_module.module().visit_with(&mut collector); + parsed_module.program().visit_with(&mut collector); assert_eq!( collector.take(), vec![ diff --git a/cli/lsp/completions.rs b/cli/lsp/completions.rs index b3d6fbbd04f655..95e5113620302e 100644 --- a/cli/lsp/completions.rs +++ b/cli/lsp/completions.rs @@ -13,11 +13,10 @@ use super::resolver::LspResolver; use super::search::PackageSearchApi; use super::tsc; +use crate::graph_util::to_node_resolution_mode; use crate::jsr::JsrFetchResolver; use crate::util::path::is_importable_ext; use crate::util::path::relative_specifier; -use deno_graph::source::ResolutionMode; -use deno_graph::Range; use deno_runtime::deno_node::SUPPORTED_BUILTIN_NODE_MODULES; use deno_ast::LineAndColumnIndex; @@ -29,12 +28,14 @@ use deno_core::serde::Serialize; use deno_core::serde_json::json; use deno_core::url::Position; use deno_core::ModuleSpecifier; -use deno_runtime::fs_util::specifier_to_file_path; +use deno_path_util::url_to_file_path; use deno_semver::jsr::JsrPackageReqReference; use deno_semver::package::PackageNv; use import_map::ImportMap; use indexmap::IndexSet; use lsp_types::CompletionList; +use node_resolver::NodeResolutionKind; +use node_resolver::ResolutionMode; use once_cell::sync::Lazy; use regex::Regex; use tower_lsp::lsp_types as lsp; @@ -111,7 +112,7 @@ async fn check_auto_config_registry( /// which we want to ignore when replacing text. fn to_narrow_lsp_range( text_info: &SourceTextInfo, - range: &deno_graph::Range, + range: deno_graph::PositionRange, ) -> lsp::Range { let end_byte_index = text_info .loc_to_source_pos(LineAndColumnIndex { @@ -164,18 +165,20 @@ pub async fn get_import_completions( ) -> Option { let document = documents.get(specifier)?; let file_referrer = document.file_referrer(); - let (text, _, range) = document.get_maybe_dependency(position)?; - let range = to_narrow_lsp_range(document.text_info(), &range); + let (text, _, graph_range) = document.get_maybe_dependency(position)?; + let resolution_mode = graph_range + .resolution_mode + .map(to_node_resolution_mode) + .unwrap_or_else(|| document.resolution_mode()); + let range = to_narrow_lsp_range(document.text_info(), graph_range.range); let resolved = resolver - .as_graph_resolver(file_referrer) + .as_cli_resolver(file_referrer) .resolve( &text, - &Range { - specifier: specifier.clone(), - start: deno_graph::Position::zeroed(), - end: deno_graph::Position::zeroed(), - }, - ResolutionMode::Execution, + specifier, + deno_graph::Position::zeroed(), + resolution_mode, + NodeResolutionKind::Execution, ) .ok(); if let Some(completion_list) = get_jsr_completions( @@ -200,15 +203,11 @@ pub async fn get_import_completions( { // completions for import map specifiers Some(lsp::CompletionResponse::List(completion_list)) - } else if text.starts_with("./") - || text.starts_with("../") - || text.starts_with('/') + } else if let Some(completion_list) = + get_local_completions(specifier, resolution_mode, &text, &range, resolver) { // completions for local relative modules - Some(lsp::CompletionResponse::List(CompletionList { - is_incomplete: false, - items: get_local_completions(specifier, &text, &range, resolver)?, - })) + Some(lsp::CompletionResponse::List(completion_list)) } else if !text.is_empty() { // completion of modules from a module registry or cache check_auto_config_registry( @@ -359,84 +358,83 @@ fn get_import_map_completions( /// Return local completions that are relative to the base specifier. fn get_local_completions( - base: &ModuleSpecifier, + referrer: &ModuleSpecifier, + resolution_mode: ResolutionMode, text: &str, range: &lsp::Range, resolver: &LspResolver, -) -> Option> { - if base.scheme() != "file" { +) -> Option { + if referrer.scheme() != "file" { return None; } - let parent = base.join(text).ok()?.join(".").ok()?; + let parent = &text[..text.char_indices().rfind(|(_, c)| *c == '/')?.0 + 1]; let resolved_parent = resolver - .as_graph_resolver(Some(base)) + .as_cli_resolver(Some(referrer)) .resolve( - parent.as_str(), - &Range { - specifier: base.clone(), - start: deno_graph::Position::zeroed(), - end: deno_graph::Position::zeroed(), - }, - ResolutionMode::Execution, + parent, + referrer, + deno_graph::Position::zeroed(), + resolution_mode, + NodeResolutionKind::Execution, ) .ok()?; - let resolved_parent_path = specifier_to_file_path(&resolved_parent).ok()?; - let raw_parent = - &text[..text.char_indices().rfind(|(_, c)| *c == '/')?.0 + 1]; + let resolved_parent_path = url_to_file_path(&resolved_parent).ok()?; if resolved_parent_path.is_dir() { let cwd = std::env::current_dir().ok()?; - let items = std::fs::read_dir(resolved_parent_path).ok()?; - Some( - items - .filter_map(|de| { - let de = de.ok()?; - let label = de.path().file_name()?.to_string_lossy().to_string(); - let entry_specifier = resolve_path(de.path().to_str()?, &cwd).ok()?; - if entry_specifier == *base { - return None; - } - let full_text = format!("{raw_parent}{label}"); - let text_edit = Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit { - range: *range, - new_text: full_text.clone(), - })); - let filter_text = Some(full_text); - match de.file_type() { - Ok(file_type) if file_type.is_dir() => Some(lsp::CompletionItem { - label, - kind: Some(lsp::CompletionItemKind::FOLDER), - detail: Some("(local)".to_string()), - filter_text, - sort_text: Some("1".to_string()), - text_edit, - commit_characters: Some( - IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect(), - ), - ..Default::default() - }), - Ok(file_type) if file_type.is_file() => { - if is_importable_ext(&de.path()) { - Some(lsp::CompletionItem { - label, - kind: Some(lsp::CompletionItemKind::FILE), - detail: Some("(local)".to_string()), - filter_text, - sort_text: Some("1".to_string()), - text_edit, - commit_characters: Some( - IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect(), - ), - ..Default::default() - }) - } else { - None - } + let entries = std::fs::read_dir(resolved_parent_path).ok()?; + let items = entries + .filter_map(|de| { + let de = de.ok()?; + let label = de.path().file_name()?.to_string_lossy().to_string(); + let entry_specifier = resolve_path(de.path().to_str()?, &cwd).ok()?; + if entry_specifier == *referrer { + return None; + } + let full_text = format!("{parent}{label}"); + let text_edit = Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit { + range: *range, + new_text: full_text.clone(), + })); + let filter_text = Some(full_text); + match de.file_type() { + Ok(file_type) if file_type.is_dir() => Some(lsp::CompletionItem { + label, + kind: Some(lsp::CompletionItemKind::FOLDER), + detail: Some("(local)".to_string()), + filter_text, + sort_text: Some("1".to_string()), + text_edit, + commit_characters: Some( + IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect(), + ), + ..Default::default() + }), + Ok(file_type) if file_type.is_file() => { + if is_importable_ext(&de.path()) { + Some(lsp::CompletionItem { + label, + kind: Some(lsp::CompletionItemKind::FILE), + detail: Some("(local)".to_string()), + filter_text, + sort_text: Some("1".to_string()), + text_edit, + commit_characters: Some( + IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect(), + ), + ..Default::default() + }) + } else { + None } - _ => None, } - }) - .collect(), - ) + _ => None, + } + }) + .collect(); + Some(CompletionList { + is_incomplete: false, + items, + }) } else { None } @@ -828,7 +826,6 @@ mod tests { use crate::lsp::documents::LanguageId; use crate::lsp::search::tests::TestPackageSearchApi; use deno_core::resolve_url; - use deno_graph::Range; use pretty_assertions::assert_eq; use std::collections::HashMap; use test_util::TempDir; @@ -909,6 +906,7 @@ mod tests { ModuleSpecifier::from_file_path(file_c).expect("could not create"); let actual = get_local_completions( &specifier, + ResolutionMode::Import, "./", &lsp::Range { start: lsp::Position { @@ -921,11 +919,11 @@ mod tests { }, }, &Default::default(), - ); - assert!(actual.is_some()); - let actual = actual.unwrap(); - assert_eq!(actual.len(), 3); - for item in actual { + ) + .unwrap(); + assert!(!actual.is_incomplete); + assert_eq!(actual.items.len(), 3); + for item in actual.items { match item.text_edit { Some(lsp::CompletionTextEdit::Edit(text_edit)) => { assert!(["./b", "./f.mjs", "./g.json"] @@ -1604,8 +1602,7 @@ mod tests { let text_info = SourceTextInfo::from_string(r#""te""#.to_string()); let range = to_narrow_lsp_range( &text_info, - &Range { - specifier: ModuleSpecifier::parse("https://deno.land").unwrap(), + deno_graph::PositionRange { start: deno_graph::Position { line: 0, character: 0, @@ -1628,8 +1625,7 @@ mod tests { let text_info = SourceTextInfo::from_string(r#""te"#.to_string()); let range = to_narrow_lsp_range( &text_info, - &Range { - specifier: ModuleSpecifier::parse("https://deno.land").unwrap(), + deno_graph::PositionRange { start: deno_graph::Position { line: 0, character: 0, diff --git a/cli/lsp/config.rs b/cli/lsp/config.rs index e55e7b734d59fa..ea77e36bcfe1c1 100644 --- a/cli/lsp/config.rs +++ b/cli/lsp/config.rs @@ -4,6 +4,7 @@ use deno_ast::MediaType; use deno_config::deno_json::DenoJsonCache; use deno_config::deno_json::FmtConfig; use deno_config::deno_json::FmtOptionsConfig; +use deno_config::deno_json::JsxImportSourceConfig; use deno_config::deno_json::LintConfig; use deno_config::deno_json::NodeModulesDirMode; use deno_config::deno_json::TestConfig; @@ -36,11 +37,12 @@ use deno_core::ModuleSpecifier; use deno_lint::linter::LintConfig as DenoLintConfig; use deno_npm::npm_rc::ResolvedNpmRc; use deno_package_json::PackageJsonCache; +use deno_path_util::url_to_file_path; use deno_runtime::deno_node::PackageJson; -use deno_runtime::fs_util::specifier_to_file_path; use indexmap::IndexSet; use lsp_types::ClientCapabilities; use std::collections::BTreeMap; +use std::collections::BTreeSet; use std::collections::HashMap; use std::ops::Deref; use std::ops::DerefMut; @@ -50,16 +52,20 @@ use std::sync::Arc; use tower_lsp::lsp_types as lsp; use super::logging::lsp_log; +use super::lsp_custom; +use super::urls::url_to_uri; use crate::args::discover_npmrc_from_workspace; use crate::args::has_flag_env_var; use crate::args::CliLockfile; +use crate::args::CliLockfileReadFromPathOptions; use crate::args::ConfigFile; use crate::args::LintFlags; use crate::args::LintOptions; use crate::cache::FastInsecureHasher; use crate::file_fetcher::FileFetcher; use crate::lsp::logging::lsp_warn; -use crate::resolver::SloppyImportsResolver; +use crate::resolver::CliSloppyImportsResolver; +use crate::resolver::SloppyImportsCachedFs; use crate::tools::lint::CliLinter; use crate::tools::lint::CliLinterOptions; use crate::tools::lint::LintRuleProvider; @@ -435,6 +441,8 @@ pub struct LanguagePreferences { pub use_aliases_for_renames: bool, #[serde(default)] pub quote_style: QuoteStyle, + #[serde(default)] + pub prefer_type_only_auto_imports: bool, } impl Default for LanguagePreferences { @@ -445,6 +453,7 @@ impl Default for LanguagePreferences { auto_import_file_exclude_patterns: vec![], use_aliases_for_renames: true, quote_style: Default::default(), + prefer_type_only_auto_imports: false, } } } @@ -801,7 +810,7 @@ impl Settings { /// Returns `None` if the value should be deferred to the presence of a /// `deno.json` file. pub fn specifier_enabled(&self, specifier: &ModuleSpecifier) -> Option { - let Ok(path) = specifier_to_file_path(specifier) else { + let Ok(path) = url_to_file_path(specifier) else { // Non-file URLs are not disabled by these settings. return Some(true); }; @@ -810,7 +819,7 @@ impl Settings { let mut disable_paths = vec![]; let mut enable_paths = None; if let Some(folder_uri) = folder_uri { - if let Ok(folder_path) = specifier_to_file_path(folder_uri) { + if let Ok(folder_path) = url_to_file_path(folder_uri) { disable_paths = settings .disable_paths .iter() @@ -847,12 +856,12 @@ impl Settings { &self, specifier: &ModuleSpecifier, ) -> (&WorkspaceSettings, Option<&ModuleSpecifier>) { - let Ok(path) = specifier_to_file_path(specifier) else { + let Ok(path) = url_to_file_path(specifier) else { return (&self.unscoped, self.first_folder.as_ref()); }; for (folder_uri, settings) in self.by_workspace_folder.iter().rev() { if let Some(settings) = settings { - let Ok(folder_path) = specifier_to_file_path(folder_uri) else { + let Ok(folder_path) = url_to_file_path(folder_uri) else { continue; }; if path.starts_with(folder_path) { @@ -977,7 +986,7 @@ impl Config { | MediaType::Tsx => Some(&workspace_settings.typescript), MediaType::Json | MediaType::Wasm - | MediaType::TsBuildInfo + | MediaType::Css | MediaType::SourceMap | MediaType::Unknown => None, } @@ -1181,8 +1190,9 @@ pub struct ConfigData { pub lockfile: Option>, pub npmrc: Option>, pub resolver: Arc, - pub sloppy_imports_resolver: Option>, + pub sloppy_imports_resolver: Option>, pub import_map_from_settings: Option, + pub unstable: BTreeSet, watched_files: HashMap, } @@ -1580,13 +1590,22 @@ impl ConfigData { .join("\n") ); } + let unstable = member_dir + .workspace + .unstable_features() + .iter() + .chain(settings.unstable.as_deref()) + .cloned() + .collect::>(); let unstable_sloppy_imports = std::env::var("DENO_UNSTABLE_SLOPPY_IMPORTS") .is_ok() - || member_dir.workspace.has_unstable("sloppy-imports"); + || unstable.contains("sloppy-imports"); let sloppy_imports_resolver = unstable_sloppy_imports.then(|| { - Arc::new(SloppyImportsResolver::new_without_stat_cache(Arc::new( - deno_runtime::deno_fs::RealFs, - ))) + Arc::new(CliSloppyImportsResolver::new( + SloppyImportsCachedFs::new_without_stat_cache(Arc::new( + deno_runtime::deno_fs::RealFs, + )), + )) }); let resolver = Arc::new(resolver); let lint_rule_provider = LintRuleProvider::new( @@ -1621,6 +1640,7 @@ impl ConfigData { lockfile, npmrc, import_map_from_settings, + unstable, watched_files, } } @@ -1635,6 +1655,17 @@ impl ConfigData { self.member_dir.maybe_pkg_json() } + pub fn maybe_jsx_import_source_config( + &self, + ) -> Option { + self + .member_dir + .workspace + .to_maybe_jsx_import_source_config() + .ok() + .flatten() + } + pub fn scope_contains_specifier(&self, specifier: &ModuleSpecifier) -> bool { specifier.as_str().starts_with(self.scope.as_str()) || self @@ -1712,14 +1743,14 @@ impl ConfigTree { .unwrap_or_else(|| Arc::new(FmtConfig::new_with_base(PathBuf::from("/")))) } - /// Returns (scope_uri, type). + /// Returns (scope_url, type). pub fn watched_file_type( &self, specifier: &ModuleSpecifier, ) -> Option<(&ModuleSpecifier, ConfigWatchedFileType)> { - for (scope_uri, data) in self.scopes.iter() { + for (scope_url, data) in self.scopes.iter() { if let Some(typ) = data.watched_files.get(specifier) { - return Some((scope_uri, *typ)); + return Some((scope_url, *typ)); } } None @@ -1743,6 +1774,46 @@ impl ConfigTree { .any(|data| data.watched_files.contains_key(specifier)) } + pub fn to_did_refresh_params( + &self, + ) -> lsp_custom::DidRefreshDenoConfigurationTreeNotificationParams { + let data = self + .scopes + .values() + .filter_map(|data| { + let workspace_root_scope_uri = + Some(data.member_dir.workspace.root_dir()) + .filter(|s| *s != data.member_dir.dir_url()) + .and_then(|s| url_to_uri(s).ok()); + Some(lsp_custom::DenoConfigurationData { + scope_uri: url_to_uri(&data.scope).ok()?, + deno_json: data.maybe_deno_json().and_then(|c| { + if workspace_root_scope_uri.is_some() + && Some(&c.specifier) + == data + .member_dir + .workspace + .root_deno_json() + .map(|c| &c.specifier) + { + return None; + } + Some(lsp::TextDocumentIdentifier { + uri: url_to_uri(&c.specifier).ok()?, + }) + }), + package_json: data.maybe_pkg_json().and_then(|p| { + Some(lsp::TextDocumentIdentifier { + uri: url_to_uri(&p.specifier()).ok()?, + }) + }), + workspace_root_scope_uri, + }) + }) + .collect(); + lsp_custom::DidRefreshDenoConfigurationTreeNotificationParams { data } + } + pub async fn refresh( &mut self, settings: &Settings, @@ -1767,7 +1838,7 @@ impl ConfigTree { let config_file_path = (|| { let config_setting = ws_settings.config.as_ref()?; let config_uri = folder_uri.join(config_setting).ok()?; - specifier_to_file_path(&config_uri).ok() + url_to_file_path(&config_uri).ok() })(); if config_file_path.is_some() || ws_settings.import_map.is_some() { scopes.insert( @@ -1844,7 +1915,7 @@ impl ConfigTree { let scope = config_file.specifier.join(".").unwrap(); let json_text = serde_json::to_string(&config_file.json).unwrap(); let test_fs = deno_runtime::deno_fs::InMemoryFs::default(); - let config_path = specifier_to_file_path(&config_file.specifier).unwrap(); + let config_path = url_to_file_path(&config_file.specifier).unwrap(); test_fs.setup_text_files(vec![( config_path.to_string_lossy().to_string(), json_text, @@ -1928,7 +1999,11 @@ fn resolve_lockfile_from_path( lockfile_path: PathBuf, frozen: bool, ) -> Option { - match CliLockfile::read_from_path(lockfile_path, frozen) { + match CliLockfile::read_from_path(CliLockfileReadFromPathOptions { + file_path: lockfile_path, + frozen, + skip_write: false, + }) { Ok(value) => { if value.filename.exists() { if let Ok(specifier) = ModuleSpecifier::from_file_path(&value.filename) @@ -2201,6 +2276,7 @@ mod tests { auto_import_file_exclude_patterns: vec![], use_aliases_for_renames: true, quote_style: QuoteStyle::Auto, + prefer_type_only_auto_imports: false, }, suggest: CompletionSettings { complete_function_calls: false, @@ -2246,6 +2322,7 @@ mod tests { auto_import_file_exclude_patterns: vec![], use_aliases_for_renames: true, quote_style: QuoteStyle::Auto, + prefer_type_only_auto_imports: false, }, suggest: CompletionSettings { complete_function_calls: false, diff --git a/cli/lsp/diagnostics.rs b/cli/lsp/diagnostics.rs index 1aebaf56fc421c..1b72953c1b8fe6 100644 --- a/cli/lsp/diagnostics.rs +++ b/cli/lsp/diagnostics.rs @@ -19,8 +19,8 @@ use super::urls::LspUrlMap; use crate::graph_util; use crate::graph_util::enhanced_resolution_error_message; use crate::lsp::lsp_custom::DiagnosticBatchNotificationParams; -use crate::resolver::SloppyImportsResolution; -use crate::resolver::SloppyImportsResolver; +use crate::resolver::CliSloppyImportsResolver; +use crate::resolver::SloppyImportsCachedFs; use crate::tools::lint::CliLinter; use crate::tools::lint::CliLinterOptions; use crate::tools::lint::LintRuleProvider; @@ -40,11 +40,12 @@ use deno_core::unsync::spawn_blocking; use deno_core::unsync::JoinHandle; use deno_core::url::Url; use deno_core::ModuleSpecifier; -use deno_graph::source::ResolutionMode; use deno_graph::source::ResolveError; use deno_graph::Resolution; use deno_graph::ResolutionError; use deno_graph::SpecifierError; +use deno_resolver::sloppy_imports::SloppyImportsResolution; +use deno_resolver::sloppy_imports::SloppyImportsResolutionKind; use deno_runtime::deno_fs; use deno_runtime::deno_node; use deno_runtime::tokio_util::create_basic_runtime; @@ -1263,7 +1264,9 @@ impl DenoDiagnostic { Self::NotInstalledJsr(pkg_req, specifier) => (lsp::DiagnosticSeverity::ERROR, format!("JSR package \"{pkg_req}\" is not installed or doesn't exist."), Some(json!({ "specifier": specifier }))), Self::NotInstalledNpm(pkg_req, specifier) => (lsp::DiagnosticSeverity::ERROR, format!("NPM package \"{pkg_req}\" is not installed or doesn't exist."), Some(json!({ "specifier": specifier }))), Self::NoLocal(specifier) => { - let maybe_sloppy_resolution = SloppyImportsResolver::new(Arc::new(deno_fs::RealFs)).resolve(specifier, ResolutionMode::Execution); + let maybe_sloppy_resolution = CliSloppyImportsResolver::new( + SloppyImportsCachedFs::new(Arc::new(deno_fs::RealFs)) + ).resolve(specifier, SloppyImportsResolutionKind::Execution); let data = maybe_sloppy_resolution.as_ref().map(|res| { json!({ "specifier": specifier, @@ -1496,7 +1499,11 @@ fn diagnose_dependency( .data_for_specifier(referrer_doc.file_referrer().unwrap_or(referrer)) .and_then(|d| d.resolver.maybe_import_map()); if let Some(import_map) = import_map { - if let Resolution::Ok(resolved) = &dependency.maybe_code { + let resolved = dependency + .maybe_code + .ok() + .or_else(|| dependency.maybe_type.ok()); + if let Some(resolved) = resolved { if let Some(to) = import_map.lookup(&resolved.specifier, referrer) { if dependency_key != to { diagnostics.push( @@ -1514,17 +1521,19 @@ fn diagnose_dependency( let import_ranges: Vec<_> = dependency .imports .iter() - .map(|i| documents::to_lsp_range(&i.range)) + .map(|i| documents::to_lsp_range(&i.specifier_range)) .collect(); // TODO(nayeemrmn): This is a crude way of detecting `@deno-types` which has // a different specifier and therefore needs a separate call to // `diagnose_resolution()`. It would be much cleaner if that were modelled as // a separate dependency: https://github.com/denoland/deno_graph/issues/247. let is_types_deno_types = !dependency.maybe_type.is_none() - && !dependency - .imports - .iter() - .any(|i| dependency.maybe_type.includes(&i.range.start).is_some()); + && !dependency.imports.iter().any(|i| { + dependency + .maybe_type + .includes(i.specifier_range.range.start) + .is_some() + }); diagnostics.extend( diagnose_resolution( diff --git a/cli/lsp/documents.rs b/cli/lsp/documents.rs index e96b8831bb960d..df51c07a39d11f 100644 --- a/cli/lsp/documents.rs +++ b/cli/lsp/documents.rs @@ -4,6 +4,8 @@ use super::cache::calculate_fs_version; use super::cache::LspCache; use super::config::Config; use super::resolver::LspResolver; +use super::resolver::ScopeDepInfo; +use super::resolver::SingleReferrerGraphResolver; use super::testing::TestCollector; use super::testing::TestModule; use super::text::LineIndex; @@ -24,18 +26,18 @@ use deno_core::futures::future::Shared; use deno_core::futures::FutureExt; use deno_core::parking_lot::Mutex; use deno_core::ModuleSpecifier; -use deno_graph::source::ResolutionMode; use deno_graph::Resolution; +use deno_path_util::url_to_file_path; use deno_runtime::deno_node; -use deno_runtime::fs_util::specifier_to_file_path; use deno_semver::jsr::JsrPackageReqReference; use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageReq; use indexmap::IndexMap; use indexmap::IndexSet; +use node_resolver::NodeResolutionKind; +use node_resolver::ResolutionMode; use std::borrow::Cow; use std::collections::BTreeMap; -use std::collections::BTreeSet; use std::collections::HashMap; use std::collections::HashSet; use std::fs; @@ -272,7 +274,7 @@ fn get_maybe_test_module_fut( parsed_source.specifier().clone(), parsed_source.text_info_lazy().clone(), ); - parsed_source.module().visit_with(&mut collector); + parsed_source.program().visit_with(&mut collector); Arc::new(collector.take()) }) .map(Result::ok) @@ -293,6 +295,8 @@ pub struct Document { /// Contains the last-known-good set of dependencies from parsing the module. config: Arc, dependencies: Arc>, + /// If this is maybe a CJS script and maybe not an ES module. + is_script: Option, // TODO(nayeemrmn): This is unused, use it for scope attribution for remote // modules. file_referrer: Option, @@ -308,6 +312,7 @@ pub struct Document { media_type: MediaType, /// Present if and only if this is an open document. open_data: Option, + resolution_mode: ResolutionMode, resolver: Arc, specifier: ModuleSpecifier, text: Arc, @@ -332,13 +337,9 @@ impl Document { .filter(|s| cache.is_valid_file_referrer(s)) .cloned() .or(file_referrer); - let media_type = resolve_media_type( - &specifier, - maybe_headers.as_ref(), - maybe_language_id, - &resolver, - ); - let (maybe_parsed_source, maybe_module) = + let media_type = + resolve_media_type(&specifier, maybe_headers.as_ref(), maybe_language_id); + let (maybe_parsed_source, maybe_module, resolution_mode) = if media_type_is_diagnosable(media_type) { parse_and_analyze_module( specifier.clone(), @@ -349,7 +350,7 @@ impl Document { &resolver, ) } else { - (None, None) + (None, None, ResolutionMode::Import) }; let maybe_module = maybe_module.and_then(Result::ok); let dependencies = maybe_module @@ -371,6 +372,7 @@ impl Document { file_referrer.as_ref(), ), file_referrer, + is_script: maybe_module.as_ref().map(|m| m.is_script), maybe_types_dependency, line_index, maybe_language_id, @@ -383,6 +385,7 @@ impl Document { maybe_parsed_source, maybe_semantic_tokens: Default::default(), }), + resolution_mode, resolver, specifier, text, @@ -399,38 +402,53 @@ impl Document { &self.specifier, self.maybe_headers.as_ref(), self.maybe_language_id, - &resolver, ); let dependencies; let maybe_types_dependency; let maybe_parsed_source; + let found_resolution_mode; + let is_script; let maybe_test_module_fut; if media_type != self.media_type { let parsed_source_result = parse_source(self.specifier.clone(), self.text.clone(), media_type); - let maybe_module = analyze_module( + let (maybe_module_result, resolution_mode) = analyze_module( self.specifier.clone(), &parsed_source_result, self.maybe_headers.as_ref(), self.file_referrer.as_ref(), &resolver, - ) - .ok(); + ); + let maybe_module = maybe_module_result.ok(); dependencies = maybe_module .as_ref() .map(|m| Arc::new(m.dependencies.clone())) .unwrap_or_default(); + is_script = maybe_module.as_ref().map(|m| m.is_script); maybe_types_dependency = maybe_module .as_ref() .and_then(|m| Some(Arc::new(m.maybe_types_dependency.clone()?))); maybe_parsed_source = Some(parsed_source_result); maybe_test_module_fut = get_maybe_test_module_fut(maybe_parsed_source.as_ref(), &config); + found_resolution_mode = resolution_mode; } else { - let graph_resolver = - resolver.as_graph_resolver(self.file_referrer.as_ref()); + let cli_resolver = resolver.as_cli_resolver(self.file_referrer.as_ref()); + let is_cjs_resolver = + resolver.as_is_cjs_resolver(self.file_referrer.as_ref()); let npm_resolver = resolver.create_graph_npm_resolver(self.file_referrer.as_ref()); + let config_data = resolver.as_config_data(self.file_referrer.as_ref()); + let jsx_import_source_config = + config_data.and_then(|d| d.maybe_jsx_import_source_config()); + found_resolution_mode = is_cjs_resolver + .get_lsp_resolution_mode(&self.specifier, self.is_script); + let resolver = SingleReferrerGraphResolver { + valid_referrer: &self.specifier, + module_resolution_mode: found_resolution_mode, + cli_resolver, + jsx_import_source_config: jsx_import_source_config.as_ref(), + }; dependencies = Arc::new( self .dependencies @@ -441,7 +459,7 @@ impl Document { d.with_new_resolver( s, &CliJsrUrlProvider, - Some(graph_resolver), + Some(&resolver), Some(&npm_resolver), ), ) @@ -451,10 +469,11 @@ impl Document { maybe_types_dependency = self.maybe_types_dependency.as_ref().map(|d| { Arc::new(d.with_new_resolver( &CliJsrUrlProvider, - Some(graph_resolver), + Some(&resolver), Some(&npm_resolver), )) }); + is_script = self.is_script; maybe_parsed_source = self.maybe_parsed_source().cloned(); maybe_test_module_fut = self .maybe_test_module_fut @@ -466,6 +485,7 @@ impl Document { // updated properties dependencies, file_referrer: self.file_referrer.clone(), + is_script, maybe_types_dependency, maybe_navigation_tree: Mutex::new(None), // maintain - this should all be copies/clones @@ -475,6 +495,7 @@ impl Document { maybe_language_id: self.maybe_language_id, maybe_test_module_fut, media_type, + resolution_mode: found_resolution_mode, open_data: self.open_data.as_ref().map(|d| DocumentOpenData { lsp_version: d.lsp_version, maybe_parsed_source, @@ -511,7 +532,7 @@ impl Document { } let text: Arc = content.into(); let media_type = self.media_type; - let (maybe_parsed_source, maybe_module) = if self + let (maybe_parsed_source, maybe_module, resolution_mode) = if self .maybe_language_id .as_ref() .map(|li| li.is_diagnosable()) @@ -526,7 +547,7 @@ impl Document { self.resolver.as_ref(), ) } else { - (None, None) + (None, None, ResolutionMode::Import) }; let maybe_module = maybe_module.and_then(Result::ok); let dependencies = maybe_module @@ -546,6 +567,7 @@ impl Document { get_maybe_test_module_fut(maybe_parsed_source.as_ref(), &self.config); Ok(Arc::new(Self { config: self.config.clone(), + is_script: maybe_module.as_ref().map(|m| m.is_script), specifier: self.specifier.clone(), file_referrer: self.file_referrer.clone(), maybe_fs_version: self.maybe_fs_version.clone(), @@ -559,6 +581,7 @@ impl Document { maybe_navigation_tree: Mutex::new(None), maybe_test_module_fut, media_type, + resolution_mode, open_data: self.open_data.is_some().then_some(DocumentOpenData { lsp_version: version, maybe_parsed_source, @@ -580,6 +603,7 @@ impl Document { ), maybe_language_id: self.maybe_language_id, dependencies: self.dependencies.clone(), + is_script: self.is_script, maybe_types_dependency: self.maybe_types_dependency.clone(), text: self.text.clone(), text_info_cell: once_cell::sync::OnceCell::new(), @@ -591,6 +615,7 @@ impl Document { maybe_test_module_fut: self.maybe_test_module_fut.clone(), media_type: self.media_type, open_data: None, + resolution_mode: self.resolution_mode, resolver: self.resolver.clone(), }) } @@ -607,6 +632,7 @@ impl Document { ), maybe_language_id: self.maybe_language_id, dependencies: self.dependencies.clone(), + is_script: self.is_script, maybe_types_dependency: self.maybe_types_dependency.clone(), text: self.text.clone(), text_info_cell: once_cell::sync::OnceCell::new(), @@ -618,6 +644,7 @@ impl Document { maybe_test_module_fut: self.maybe_test_module_fut.clone(), media_type: self.media_type, open_data: self.open_data.clone(), + resolution_mode: self.resolution_mode, resolver: self.resolver.clone(), }) } @@ -641,6 +668,10 @@ impl Document { &self.text } + pub fn resolution_mode(&self) -> ResolutionMode { + self.resolution_mode + } + pub fn text_info(&self) -> &SourceTextInfo { // try to get the text info from the parsed source and if // not then create one in the cell @@ -654,7 +685,6 @@ impl Document { .get_or_init(|| SourceTextInfo::new(self.text.clone())) }) } - pub fn line_index(&self) -> Arc { self.line_index.clone() } @@ -738,7 +768,7 @@ impl Document { }; self.dependencies().iter().find_map(|(s, dep)| { dep - .includes(&position) + .includes(position) .map(|r| (s.clone(), dep.clone(), r.clone())) }) } @@ -764,14 +794,7 @@ fn resolve_media_type( specifier: &ModuleSpecifier, maybe_headers: Option<&HashMap>, maybe_language_id: Option, - resolver: &LspResolver, ) -> MediaType { - if resolver.in_node_modules(specifier) { - if let Some(media_type) = resolver.node_media_type(specifier) { - return media_type; - } - } - if let Some(language_id) = maybe_language_id { return MediaType::from_specifier_and_content_type( specifier, @@ -786,15 +809,15 @@ fn resolve_media_type( MediaType::from_specifier(specifier) } -pub fn to_lsp_range(range: &deno_graph::Range) -> lsp::Range { +pub fn to_lsp_range(referrer: &deno_graph::Range) -> lsp::Range { lsp::Range { start: lsp::Position { - line: range.start.line as u32, - character: range.start.character as u32, + line: referrer.range.start.line as u32, + character: referrer.range.start.character as u32, }, end: lsp::Position { - line: range.end.line as u32, - character: range.end.character as u32, + line: referrer.range.end.line as u32, + character: referrer.range.end.character as u32, }, } } @@ -849,10 +872,15 @@ impl FileSystemDocuments { file_referrer: Option<&ModuleSpecifier>, ) -> Option> { let doc = if specifier.scheme() == "file" { - let path = specifier_to_file_path(specifier).ok()?; + let path = url_to_file_path(specifier).ok()?; let bytes = fs::read(path).ok()?; - let content = - deno_graph::source::decode_owned_source(specifier, bytes, None).ok()?; + let content = bytes_to_content( + specifier, + MediaType::from_specifier(specifier), + bytes, + None, + ) + .ok()?; Document::new( specifier.clone(), content.into(), @@ -889,19 +917,24 @@ impl FileSystemDocuments { specifier, Some(&cached_file.metadata.headers), ); - let content = deno_graph::source::decode_owned_source( + let media_type = resolve_media_type( + specifier, + Some(&cached_file.metadata.headers), + None, + ); + let content = bytes_to_content( specifier, + media_type, cached_file.content, maybe_charset, ) .ok()?; - let maybe_headers = Some(cached_file.metadata.headers); Document::new( specifier.clone(), content.into(), None, None, - maybe_headers, + Some(cached_file.metadata.headers), resolver.clone(), config.clone(), cache, @@ -942,6 +975,9 @@ pub struct Documents { /// The DENO_DIR that the documents looks for non-file based modules. cache: Arc, config: Arc, + /// A resolver that takes into account currently loaded import map and JSX + /// settings. + resolver: Arc, /// A flag that indicates that stated data is potentially invalid and needs to /// be recalculated before being considered valid. dirty: bool, @@ -949,15 +985,7 @@ pub struct Documents { open_docs: HashMap>, /// Documents stored on the file system. file_system_docs: Arc, - /// A resolver that takes into account currently loaded import map and JSX - /// settings. - resolver: Arc, - /// The npm package requirements found in npm specifiers. - npm_reqs_by_scope: - Arc, BTreeSet>>, - /// Config scopes that contain a node: specifier such that a @types/node - /// package should be injected. - scopes_with_node_specifier: Arc>>, + dep_info_by_scope: Arc, Arc>>, } impl Documents { @@ -1071,34 +1099,6 @@ impl Documents { self.cache.is_valid_file_referrer(specifier) } - /// Return `true` if the provided specifier can be resolved to a document, - /// otherwise `false`. - pub fn contains_import( - &self, - specifier: &str, - referrer: &ModuleSpecifier, - ) -> bool { - let file_referrer = self.get_file_referrer(referrer); - let maybe_specifier = self - .resolver - .as_graph_resolver(file_referrer.as_deref()) - .resolve( - specifier, - &deno_graph::Range { - specifier: referrer.clone(), - start: deno_graph::Position::zeroed(), - end: deno_graph::Position::zeroed(), - }, - ResolutionMode::Types, - ) - .ok(); - if let Some(import_specifier) = maybe_specifier { - self.exists(&import_specifier, file_referrer.as_deref()) - } else { - false - } - } - pub fn resolve_document_specifier( &self, specifier: &ModuleSpecifier, @@ -1136,7 +1136,7 @@ impl Documents { return true; } if specifier.scheme() == "file" { - return specifier_to_file_path(&specifier) + return url_to_file_path(&specifier) .map(|p| p.is_file()) .unwrap_or(false); } @@ -1147,17 +1147,20 @@ impl Documents { false } - pub fn npm_reqs_by_scope( + pub fn dep_info_by_scope( &mut self, - ) -> Arc, BTreeSet>> { - self.calculate_npm_reqs_if_dirty(); - self.npm_reqs_by_scope.clone() + ) -> Arc, Arc>> { + self.calculate_dep_info_if_dirty(); + self.dep_info_by_scope.clone() } - pub fn scopes_with_node_specifier( - &self, - ) -> &Arc>> { - &self.scopes_with_node_specifier + pub fn scopes_with_node_specifier(&self) -> HashSet> { + self + .dep_info_by_scope + .iter() + .filter(|(_, i)| i.has_node_specifier) + .map(|(s, _)| s.clone()) + .collect::>() } /// Return a document for the specifier. @@ -1251,18 +1254,23 @@ impl Documents { /// tsc when type checking. pub fn resolve( &self, - raw_specifiers: &[String], + // (is_cjs: bool, raw_specifier: String) + raw_specifiers: &[(bool, String)], referrer: &ModuleSpecifier, file_referrer: Option<&ModuleSpecifier>, ) -> Vec> { - let document = self.get(referrer); - let file_referrer = document + let referrer_doc = self.get(referrer); + let file_referrer = referrer_doc .as_ref() .and_then(|d| d.file_referrer()) .or(file_referrer); - let dependencies = document.as_ref().map(|d| d.dependencies()); + let dependencies = referrer_doc.as_ref().map(|d| d.dependencies()); let mut results = Vec::new(); - for raw_specifier in raw_specifiers { + for (is_cjs, raw_specifier) in raw_specifiers { + let resolution_mode = match is_cjs { + true => ResolutionMode::Require, + false => ResolutionMode::Import, + }; if raw_specifier.starts_with("asset:") { if let Ok(specifier) = ModuleSpecifier::parse(raw_specifier) { let media_type = MediaType::from_specifier(&specifier); @@ -1277,31 +1285,32 @@ impl Documents { results.push(self.resolve_dependency( specifier, referrer, + resolution_mode, file_referrer, )); } else if let Some(specifier) = dep.maybe_code.maybe_specifier() { results.push(self.resolve_dependency( specifier, referrer, + resolution_mode, file_referrer, )); } else { results.push(None); } } else if let Ok(specifier) = - self.resolver.as_graph_resolver(file_referrer).resolve( + self.resolver.as_cli_resolver(file_referrer).resolve( raw_specifier, - &deno_graph::Range { - specifier: referrer.clone(), - start: deno_graph::Position::zeroed(), - end: deno_graph::Position::zeroed(), - }, - ResolutionMode::Types, + referrer, + deno_graph::Position::zeroed(), + resolution_mode, + NodeResolutionKind::Types, ) { results.push(self.resolve_dependency( &specifier, referrer, + resolution_mode, file_referrer, )); } else { @@ -1321,11 +1330,14 @@ impl Documents { self.config = Arc::new(config.clone()); self.cache = Arc::new(cache.clone()); self.resolver = resolver.clone(); + + node_resolver::PackageJsonThreadLocalCache::clear(); + { let fs_docs = &self.file_system_docs; // Clean up non-existent documents. fs_docs.docs.retain(|specifier, _| { - let Ok(path) = specifier_to_file_path(specifier) else { + let Ok(path) = url_to_file_path(specifier) else { // Remove non-file schemed docs (deps). They may not be dependencies // anymore after updating resolvers. return false; @@ -1379,34 +1391,46 @@ impl Documents { /// Iterate through the documents, building a map where the key is a unique /// document and the value is a set of specifiers that depend on that /// document. - fn calculate_npm_reqs_if_dirty(&mut self) { - let mut npm_reqs_by_scope: BTreeMap<_, BTreeSet<_>> = Default::default(); - let mut scopes_with_specifier = HashSet::new(); + fn calculate_dep_info_if_dirty(&mut self) { + let mut dep_info_by_scope: BTreeMap<_, ScopeDepInfo> = Default::default(); let is_fs_docs_dirty = self.file_system_docs.set_dirty(false); if !is_fs_docs_dirty && !self.dirty { return; } let mut visit_doc = |doc: &Arc| { let scope = doc.scope(); - let reqs = npm_reqs_by_scope.entry(scope.cloned()).or_default(); + let dep_info = dep_info_by_scope.entry(scope.cloned()).or_default(); for dependency in doc.dependencies().values() { - if let Some(dep) = dependency.get_code() { + let code_specifier = dependency.get_code(); + let type_specifier = dependency.get_type(); + if let Some(dep) = code_specifier { if dep.scheme() == "node" { - scopes_with_specifier.insert(scope.cloned()); + dep_info.has_node_specifier = true; } if let Ok(reference) = NpmPackageReqReference::from_specifier(dep) { - reqs.insert(reference.into_inner().req); + dep_info.npm_reqs.insert(reference.into_inner().req); } } - if let Some(dep) = dependency.get_type() { + if let Some(dep) = type_specifier { if let Ok(reference) = NpmPackageReqReference::from_specifier(dep) { - reqs.insert(reference.into_inner().req); + dep_info.npm_reqs.insert(reference.into_inner().req); + } + } + if dependency.maybe_deno_types_specifier.is_some() { + if let (Some(code_specifier), Some(type_specifier)) = + (code_specifier, type_specifier) + { + if MediaType::from_specifier(type_specifier).is_declaration() { + dep_info + .deno_types_to_code_resolutions + .insert(type_specifier.clone(), code_specifier.clone()); + } } } } if let Some(dep) = doc.maybe_types_dependency().maybe_specifier() { if let Ok(reference) = NpmPackageReqReference::from_specifier(dep) { - reqs.insert(reference.into_inner().req); + dep_info.npm_reqs.insert(reference.into_inner().req); } } }; @@ -1417,14 +1441,46 @@ impl Documents { visit_doc(doc); } - // fill the reqs from the lockfile for (scope, config_data) in self.config.tree.data_by_scope().as_ref() { + let dep_info = dep_info_by_scope.entry(Some(scope.clone())).or_default(); + (|| { + let config_file = config_data.maybe_deno_json()?; + let jsx_config = + config_file.to_maybe_jsx_import_source_config().ok()??; + let type_specifier = jsx_config.default_types_specifier.as_ref()?; + let code_specifier = jsx_config.default_specifier.as_ref()?; + let cli_resolver = self.resolver.as_cli_resolver(Some(scope)); + let type_specifier = cli_resolver + .resolve( + type_specifier, + &jsx_config.base_url, + deno_graph::Position::zeroed(), + // todo(dsherret): this is wrong because it doesn't consider CJS referrers + ResolutionMode::Import, + NodeResolutionKind::Types, + ) + .ok()?; + let code_specifier = cli_resolver + .resolve( + code_specifier, + &jsx_config.base_url, + deno_graph::Position::zeroed(), + // todo(dsherret): this is wrong because it doesn't consider CJS referrers + ResolutionMode::Import, + NodeResolutionKind::Execution, + ) + .ok()?; + dep_info + .deno_types_to_code_resolutions + .insert(type_specifier, code_specifier); + Some(()) + })(); + // fill the reqs from the lockfile if let Some(lockfile) = config_data.lockfile.as_ref() { - let reqs = npm_reqs_by_scope.entry(Some(scope.clone())).or_default(); let lockfile = lockfile.lock(); for dep_req in lockfile.content.packages.specifiers.keys() { if dep_req.kind == deno_semver::package::PackageKind::Npm { - reqs.insert(dep_req.req.clone()); + dep_info.npm_reqs.insert(dep_req.req.clone()); } } } @@ -1433,15 +1489,22 @@ impl Documents { // Ensure a @types/node package exists when any module uses a node: specifier. // Unlike on the command line, here we just add @types/node to the npm package // requirements since this won't end up in the lockfile. - for scope in &scopes_with_specifier { - let reqs = npm_reqs_by_scope.entry(scope.clone()).or_default(); - if !reqs.iter().any(|r| r.name == "@types/node") { - reqs.insert(PackageReq::from_str("@types/node").unwrap()); + for dep_info in dep_info_by_scope.values_mut() { + if dep_info.has_node_specifier + && !dep_info.npm_reqs.iter().any(|r| r.name == "@types/node") + { + dep_info + .npm_reqs + .insert(PackageReq::from_str("@types/node").unwrap()); } } - self.npm_reqs_by_scope = Arc::new(npm_reqs_by_scope); - self.scopes_with_node_specifier = Arc::new(scopes_with_specifier); + self.dep_info_by_scope = Arc::new( + dep_info_by_scope + .into_iter() + .map(|(s, i)| (s, Arc::new(i))) + .collect(), + ); self.dirty = false; } @@ -1449,6 +1512,7 @@ impl Documents { &self, specifier: &ModuleSpecifier, referrer: &ModuleSpecifier, + resolution_mode: ResolutionMode, file_referrer: Option<&ModuleSpecifier>, ) -> Option<(ModuleSpecifier, MediaType)> { if let Some(module_name) = specifier.as_str().strip_prefix("node:") { @@ -1462,10 +1526,12 @@ impl Documents { let mut specifier = specifier.clone(); let mut media_type = None; if let Ok(npm_ref) = NpmPackageReqReference::from_specifier(&specifier) { - let (s, mt) = - self - .resolver - .npm_to_file_url(&npm_ref, referrer, file_referrer)?; + let (s, mt) = self.resolver.npm_to_file_url( + &npm_ref, + referrer, + resolution_mode, + file_referrer, + )?; specifier = s; media_type = Some(mt); } @@ -1475,7 +1541,12 @@ impl Documents { return Some((specifier, media_type)); }; if let Some(types) = doc.maybe_types_dependency().maybe_specifier() { - self.resolve_dependency(types, &specifier, file_referrer) + self.resolve_dependency( + types, + &specifier, + doc.resolution_mode(), + file_referrer, + ) } else { Some((doc.specifier().clone(), doc.media_type())) } @@ -1544,16 +1615,24 @@ fn parse_and_analyze_module( media_type: MediaType, file_referrer: Option<&ModuleSpecifier>, resolver: &LspResolver, -) -> (Option, Option) { +) -> ( + Option, + Option, + ResolutionMode, +) { let parsed_source_result = parse_source(specifier.clone(), text, media_type); - let module_result = analyze_module( + let (module_result, resolution_mode) = analyze_module( specifier, &parsed_source_result, maybe_headers, file_referrer, resolver, ); - (Some(parsed_source_result), Some(module_result)) + ( + Some(parsed_source_result), + Some(module_result), + resolution_mode, + ) } fn parse_source( @@ -1561,7 +1640,7 @@ fn parse_source( text: Arc, media_type: MediaType, ) -> ParsedSourceResult { - deno_ast::parse_module(deno_ast::ParseParams { + deno_ast::parse_program(deno_ast::ParseParams { specifier, text, media_type, @@ -1577,28 +1656,68 @@ fn analyze_module( maybe_headers: Option<&HashMap>, file_referrer: Option<&ModuleSpecifier>, resolver: &LspResolver, -) -> ModuleResult { +) -> (ModuleResult, ResolutionMode) { match parsed_source_result { Ok(parsed_source) => { let npm_resolver = resolver.create_graph_npm_resolver(file_referrer); - Ok(deno_graph::parse_module_from_ast( - deno_graph::ParseModuleFromAstOptions { - graph_kind: deno_graph::GraphKind::TypesOnly, - specifier, - maybe_headers, - parsed_source, - // use a null file system because there's no need to bother resolving - // dynamic imports like import(`./dir/${something}`) in the LSP - file_system: &deno_graph::source::NullFileSystem, - jsr_url_provider: &CliJsrUrlProvider, - maybe_resolver: Some(resolver.as_graph_resolver(file_referrer)), - maybe_npm_resolver: Some(&npm_resolver), - }, - )) + let cli_resolver = resolver.as_cli_resolver(file_referrer); + let is_cjs_resolver = resolver.as_is_cjs_resolver(file_referrer); + let config_data = resolver.as_config_data(file_referrer); + let valid_referrer = specifier.clone(); + let jsx_import_source_config = + config_data.and_then(|d| d.maybe_jsx_import_source_config()); + let module_resolution_mode = is_cjs_resolver.get_lsp_resolution_mode( + &specifier, + Some(parsed_source.compute_is_script()), + ); + let resolver = SingleReferrerGraphResolver { + valid_referrer: &valid_referrer, + module_resolution_mode, + cli_resolver, + jsx_import_source_config: jsx_import_source_config.as_ref(), + }; + ( + Ok(deno_graph::parse_module_from_ast( + deno_graph::ParseModuleFromAstOptions { + graph_kind: deno_graph::GraphKind::TypesOnly, + specifier, + maybe_headers, + parsed_source, + // use a null file system because there's no need to bother resolving + // dynamic imports like import(`./dir/${something}`) in the LSP + file_system: &deno_graph::source::NullFileSystem, + jsr_url_provider: &CliJsrUrlProvider, + maybe_resolver: Some(&resolver), + maybe_npm_resolver: Some(&npm_resolver), + }, + )), + module_resolution_mode, + ) } - Err(err) => Err(deno_graph::ModuleGraphError::ModuleError( - deno_graph::ModuleError::ParseErr(specifier, err.clone()), - )), + Err(err) => ( + Err(deno_graph::ModuleGraphError::ModuleError( + deno_graph::ModuleError::ParseErr(specifier, err.clone()), + )), + ResolutionMode::Import, + ), + } +} + +fn bytes_to_content( + specifier: &ModuleSpecifier, + media_type: MediaType, + bytes: Vec, + maybe_charset: Option<&str>, +) -> Result { + if media_type == MediaType::Wasm { + // we use the dts representation for Wasm modules + Ok(deno_graph::source::wasm::wasm_module_to_dts(&bytes)?) + } else { + Ok(deno_graph::source::decode_owned_source( + specifier, + bytes, + maybe_charset, + )?) } } diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index 85daa4e289bf30..0caaa941072bb6 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -15,13 +15,15 @@ use deno_core::url::Url; use deno_core::ModuleSpecifier; use deno_graph::GraphKind; use deno_graph::Resolution; +use deno_path_util::url_to_file_path; use deno_runtime::deno_tls::rustls::RootCertStore; use deno_runtime::deno_tls::RootCertStoreProvider; -use deno_runtime::fs_util::specifier_to_file_path; use deno_semver::jsr::JsrPackageReqReference; use indexmap::Equivalent; use indexmap::IndexSet; use log::error; +use node_resolver::NodeResolutionKind; +use node_resolver::ResolutionMode; use serde::Deserialize; use serde_json::from_value; use std::collections::BTreeMap; @@ -96,6 +98,7 @@ use crate::args::CaData; use crate::args::CacheSetting; use crate::args::CliOptions; use crate::args::Flags; +use crate::args::InternalFlags; use crate::args::UnstableFmtOptions; use crate::factory::CliFactory; use crate::file_fetcher::FileFetcher; @@ -207,11 +210,11 @@ pub struct Inner { module_registry: ModuleRegistry, /// A lazily create "server" for handling test run requests. maybe_testing_server: Option, - npm_search_api: CliNpmSearchApi, + pub npm_search_api: CliNpmSearchApi, project_version: usize, /// A collection of measurements which instrument that performance of the LSP. performance: Arc, - resolver: Arc, + pub resolver: Arc, task_queue: LanguageServerTaskQueue, /// A memoized version of fixable diagnostic codes retrieved from TypeScript. ts_fixable_diagnostics: Vec, @@ -274,10 +277,9 @@ impl LanguageServer { factory.fs(), &roots, graph_util::GraphValidOptions { - is_vendoring: false, kind: GraphKind::All, check_js: false, - exit_lockfile_errors: false, + exit_integrity_errors: false, }, )?; @@ -627,7 +629,7 @@ impl Inner { let maybe_root_path = self .config .root_uri() - .and_then(|uri| specifier_to_file_path(uri).ok()); + .and_then(|uri| url_to_file_path(uri).ok()); let root_cert_store = get_root_cert_store( maybe_root_path, workspace_settings.certificate_stores.clone(), @@ -803,7 +805,7 @@ impl Inner { let mut roots = config .workspace_folders .iter() - .filter_map(|p| specifier_to_file_path(&p.0).ok()) + .filter_map(|p| url_to_file_path(&p.0).ok()) .collect::>(); roots.sort(); let roots = roots @@ -863,7 +865,10 @@ impl Inner { // We ignore these directories by default because there is a // high likelihood they aren't relevant. Someone can opt-into // them by specifying one of them as an enabled path. - if matches!(dir_name.as_str(), "vendor" | "node_modules" | ".git") { + if matches!( + dir_name.as_str(), + "vendor" | "coverage" | "node_modules" | ".git" + ) { continue; } // ignore cargo target directories for anyone using Deno with Rust @@ -904,7 +909,7 @@ impl Inner { | MediaType::Tsx => {} MediaType::Wasm | MediaType::SourceMap - | MediaType::TsBuildInfo + | MediaType::Css | MediaType::Unknown => { if path.extension().and_then(|s| s.to_str()) != Some("jsonc") { continue; @@ -963,6 +968,11 @@ impl Inner { .tree .refresh(&self.config.settings, &self.workspace_files, &file_fetcher) .await; + self + .client + .send_did_refresh_deno_configuration_tree_notification( + self.config.tree.to_did_refresh_params(), + ); for config_file in self.config.tree.config_files() { (|| { let compiler_options = config_file.to_compiler_options().ok()?.options; @@ -974,15 +984,13 @@ impl Inner { spawn(async move { let specifier = { let inner = ls.inner.read().await; - let resolver = inner.resolver.as_graph_resolver(Some(&referrer)); + let resolver = inner.resolver.as_cli_resolver(Some(&referrer)); let Ok(specifier) = resolver.resolve( &specifier, - &deno_graph::Range { - specifier: referrer.clone(), - start: deno_graph::Position::zeroed(), - end: deno_graph::Position::zeroed(), - }, - deno_graph::source::ResolutionMode::Types, + &referrer, + deno_graph::Position::zeroed(), + ResolutionMode::Import, + NodeResolutionKind::Types, ) else { return; }; @@ -1019,7 +1027,7 @@ impl Inner { // refresh the npm specifiers because it might have discovered // a @types/node package and now's a good time to do that anyway - self.refresh_npm_specifiers().await; + self.refresh_dep_info().await; self.project_changed([], true); } @@ -1065,7 +1073,7 @@ impl Inner { ); if document.is_diagnosable() { self.project_changed([(document.specifier(), ChangeKind::Opened)], false); - self.refresh_npm_specifiers().await; + self.refresh_dep_info().await; self.diagnostics_server.invalidate(&[specifier]); self.send_diagnostics_update(); self.send_testing_update(); @@ -1086,8 +1094,8 @@ impl Inner { Ok(document) => { if document.is_diagnosable() { let old_scopes_with_node_specifier = - self.documents.scopes_with_node_specifier().clone(); - self.refresh_npm_specifiers().await; + self.documents.scopes_with_node_specifier(); + self.refresh_dep_info().await; let mut config_changed = false; if !self .documents @@ -1125,7 +1133,7 @@ impl Inner { { return; } - match specifier_to_file_path(&specifier) { + match url_to_file_path(&specifier) { Ok(path) if is_importable_ext(&path) => {} _ => return, } @@ -1138,13 +1146,15 @@ impl Inner { })); } - async fn refresh_npm_specifiers(&mut self) { - let package_reqs = self.documents.npm_reqs_by_scope(); + async fn refresh_dep_info(&mut self) { + let dep_info_by_scope = self.documents.dep_info_by_scope(); let resolver = self.resolver.clone(); // spawn due to the lsp's `Send` requirement - spawn(async move { resolver.set_npm_reqs(&package_reqs).await }) - .await - .ok(); + spawn( + async move { resolver.set_dep_info_by_scope(&dep_info_by_scope).await }, + ) + .await + .ok(); } async fn did_close(&mut self, params: DidCloseTextDocumentParams) { @@ -1163,7 +1173,7 @@ impl Inner { .uri_to_specifier(¶ms.text_document.uri, LspUrlKind::File); self.diagnostics_state.clear(&specifier); if self.is_diagnosable(&specifier) { - self.refresh_npm_specifiers().await; + self.refresh_dep_info().await; self.diagnostics_server.invalidate(&[specifier.clone()]); self.send_diagnostics_update(); self.send_testing_update(); @@ -1363,7 +1373,7 @@ impl Inner { { specifier = uri_to_url(¶ms.text_document.uri); } - let file_path = specifier_to_file_path(&specifier).map_err(|err| { + let file_path = url_to_file_path(&specifier).map_err(|err| { error!("{:#}", err); LspError::invalid_request() })?; @@ -1377,16 +1387,17 @@ impl Inner { .fmt_config_for_specifier(&specifier) .options .clone(); - fmt_options.use_tabs = Some(!params.options.insert_spaces); - fmt_options.indent_width = Some(params.options.tab_size as u8); - let maybe_workspace = self - .config - .tree - .data_for_specifier(&specifier) - .map(|d| &d.member_dir.workspace); + let config_data = self.config.tree.data_for_specifier(&specifier); + if !config_data.is_some_and(|d| d.maybe_deno_json().is_some()) { + fmt_options.use_tabs = Some(!params.options.insert_spaces); + fmt_options.indent_width = Some(params.options.tab_size as u8); + } let unstable_options = UnstableFmtOptions { - component: maybe_workspace - .map(|w| w.has_unstable("fmt-component")) + component: config_data + .map(|d| d.unstable.contains("fmt-component")) + .unwrap_or(false), + sql: config_data + .map(|d| d.unstable.contains("fmt-sql")) .unwrap_or(false), }; let document = document.clone(); @@ -1613,11 +1624,15 @@ impl Inner { None => false, }) .collect(); + let mut code_actions = CodeActionCollection::default(); if !fixable_diagnostics.is_empty() { - let mut code_actions = CodeActionCollection::default(); let file_diagnostics = self .diagnostics_server .get_ts_diagnostics(&specifier, asset_or_doc.document_lsp_version()); + let specifier_kind = asset_or_doc + .document() + .map(|d| d.resolution_mode()) + .unwrap_or(ResolutionMode::Import); let mut includes_no_cache = false; for diagnostic in &fixable_diagnostics { match diagnostic.source.as_deref() { @@ -1656,7 +1671,13 @@ impl Inner { .await; for action in actions { code_actions - .add_ts_fix_action(&specifier, &action, diagnostic, self) + .add_ts_fix_action( + &specifier, + specifier_kind, + &action, + diagnostic, + self, + ) .map_err(|err| { error!("Unable to convert fix: {:#}", err); LspError::internal_error() @@ -1722,9 +1743,14 @@ impl Inner { .add_cache_all_action(&specifier, no_cache_diagnostics.to_owned()); } } - code_actions.set_preferred_fixes(); - all_actions.extend(code_actions.get_response()); } + if let Some(document) = asset_or_doc.document() { + code_actions + .add_source_actions(document, ¶ms.range, self) + .await; + } + code_actions.set_preferred_fixes(); + all_actions.extend(code_actions.get_response()); // Refactor let only = params @@ -1797,10 +1823,9 @@ impl Inner { error!("Unable to decode code action data: {:#}", err); LspError::invalid_params("The CodeAction's data is invalid.") })?; - let scope = self - .get_asset_or_document(&code_action_data.specifier) - .ok() - .and_then(|d| d.scope().cloned()); + let maybe_asset_or_doc = + self.get_asset_or_document(&code_action_data.specifier).ok(); + let scope = maybe_asset_or_doc.as_ref().and_then(|d| d.scope().cloned()); let combined_code_actions = self .ts_server .get_combined_code_fix( @@ -1827,8 +1852,13 @@ impl Inner { let changes = if code_action_data.fix_id == "fixMissingImport" { fix_ts_import_changes( &code_action_data.specifier, + maybe_asset_or_doc + .as_ref() + .and_then(|d| d.document()) + .map(|d| d.resolution_mode()) + .unwrap_or(ResolutionMode::Import), &combined_code_actions.changes, - &self.get_ts_response_import_mapper(&code_action_data.specifier), + self, ) .map_err(|err| { error!("Unable to remap changes: {:#}", err); @@ -1880,8 +1910,12 @@ impl Inner { if kind_suffix == ".rewrite.function.returnType" { refactor_edit_info.edits = fix_ts_import_changes( &action_data.specifier, + asset_or_doc + .document() + .map(|d| d.resolution_mode()) + .unwrap_or(ResolutionMode::Import), &refactor_edit_info.edits, - &self.get_ts_response_import_mapper(&action_data.specifier), + self, ) .map_err(|err| { error!("Unable to remap changes: {:#}", err); @@ -1912,7 +1946,9 @@ impl Inner { // todo(dsherret): this should probably just take the resolver itself // as the import map is an implementation detail .and_then(|d| d.resolver.maybe_import_map()), - self.resolver.as_ref(), + &self.resolver, + &self.ts_server.specifier_map, + file_referrer, ) } @@ -2274,7 +2310,11 @@ impl Inner { .into(), scope.cloned(), ) - .await; + .await + .unwrap_or_else(|err| { + error!("Unable to get completion info from TypeScript: {:#}", err); + None + }); if let Some(completions) = maybe_completion_info { response = Some( @@ -2509,7 +2549,7 @@ impl Inner { let maybe_root_path_owned = self .config .root_uri() - .and_then(|uri| specifier_to_file_path(uri).ok()); + .and_then(|uri| url_to_file_path(uri).ok()); let mut resolved_items = Vec::::new(); for item in incoming_calls.iter() { if let Some(resolved) = item.try_resolve_call_hierarchy_incoming_call( @@ -2555,7 +2595,7 @@ impl Inner { let maybe_root_path_owned = self .config .root_uri() - .and_then(|uri| specifier_to_file_path(uri).ok()); + .and_then(|uri| url_to_file_path(uri).ok()); let mut resolved_items = Vec::::new(); for item in outgoing_calls.iter() { if let Some(resolved) = item.try_resolve_call_hierarchy_outgoing_call( @@ -2604,7 +2644,7 @@ impl Inner { let maybe_root_path_owned = self .config .root_uri() - .and_then(|uri| specifier_to_file_path(uri).ok()); + .and_then(|uri| url_to_file_path(uri).ok()); let mut resolved_items = Vec::::new(); match one_or_many { tsc::OneOrMany::One(item) => { @@ -3557,15 +3597,16 @@ impl Inner { if byonm { roots.retain(|s| s.scheme() != "npm"); - } else if let Some(npm_reqs) = self + } else if let Some(dep_info) = self .documents - .npm_reqs_by_scope() + .dep_info_by_scope() .get(&config_data.map(|d| d.scope.as_ref().clone())) { // always include the npm packages since resolution of one npm package // might affect the resolution of other npm packages roots.extend( - npm_reqs + dep_info + .npm_reqs .iter() .map(|req| ModuleSpecifier::parse(&format!("npm:{}", req)).unwrap()), ); @@ -3586,9 +3627,8 @@ impl Inner { deno_json_cache: None, pkg_json_cache: None, workspace_cache: None, - config_parse_options: deno_config::deno_json::ConfigParseOptions { - include_task_comments: false, - }, + config_parse_options: + deno_config::deno_json::ConfigParseOptions::default(), additional_config_file_names: &[], discover_pkg_json: !has_flag_env_var("DENO_NO_PACKAGE_JSON"), maybe_vendor_override: if force_global_cache { @@ -3601,7 +3641,10 @@ impl Inner { }; let cli_options = CliOptions::new( Arc::new(Flags { - cache_path: Some(self.cache.deno_dir().root.clone()), + internal: InternalFlags { + cache_path: Some(self.cache.deno_dir().root.clone()), + ..Default::default() + }, ca_stores: workspace_settings.certificate_stores.clone(), ca_data: workspace_settings.tls_certificate.clone().map(CaData::File), unsafely_ignore_certificate_errors: workspace_settings @@ -3614,6 +3657,11 @@ impl Inner { }), // bit of a hack to force the lsp to cache the @types/node package type_check_mode: crate::args::TypeCheckMode::Local, + permissions: crate::args::PermissionFlags { + // allow remote import permissions in the lsp for now + allow_import: Some(vec![]), + ..Default::default() + }, ..Default::default() }), initial_cwd, @@ -3635,7 +3683,7 @@ impl Inner { async fn post_cache(&mut self) { self.resolver.did_cache(); - self.refresh_npm_specifiers().await; + self.refresh_dep_info().await; self.diagnostics_server.invalidate_all(); self.project_changed([], true); self.ts_server.cleanup_semantic_cache(self.snapshot()).await; @@ -3723,14 +3771,11 @@ impl Inner { fn task_definitions(&self) -> LspResult> { let mut result = vec![]; for config_file in self.config.tree.config_files() { - if let Some(tasks) = json!(&config_file.json.tasks).as_object() { - for (name, value) in tasks { - let Some(command) = value.as_str() else { - continue; - }; + if let Some(tasks) = config_file.to_tasks_config().ok().flatten() { + for (name, def) in tasks { result.push(TaskDefinition { name: name.clone(), - command: command.to_string(), + command: def.command.clone(), source_uri: url_to_uri(&config_file.specifier) .map_err(|_| LspError::internal_error())?, }); @@ -3793,7 +3838,7 @@ impl Inner { let maybe_inlay_hints = maybe_inlay_hints.map(|hints| { hints .iter() - .map(|hint| hint.to_lsp(line_index.clone())) + .map(|hint| hint.to_lsp(line_index.clone(), self)) .collect() }); self.performance.measure(mark); @@ -3929,7 +3974,9 @@ mod tests { fn test_walk_workspace() { let temp_dir = TempDir::new(); temp_dir.create_dir_all("root1/vendor/"); + temp_dir.create_dir_all("root1/coverage/"); temp_dir.write("root1/vendor/mod.ts", ""); // no, vendor + temp_dir.write("root1/coverage/mod.ts", ""); // no, coverage temp_dir.create_dir_all("root1/node_modules/"); temp_dir.write("root1/node_modules/mod.ts", ""); // no, node_modules diff --git a/cli/lsp/lsp_custom.rs b/cli/lsp/lsp_custom.rs index 5f485db7acd1b1..74c6aca88b5b5a 100644 --- a/cli/lsp/lsp_custom.rs +++ b/cli/lsp/lsp_custom.rs @@ -14,8 +14,6 @@ pub const LATEST_DIAGNOSTIC_BATCH_INDEX: &str = #[serde(rename_all = "camelCase")] pub struct TaskDefinition { pub name: String, - // TODO(nayeemrmn): Rename this to `command` in vscode_deno. - #[serde(rename = "detail")] pub command: String, pub source_uri: lsp::Uri, } @@ -46,6 +44,30 @@ pub struct DiagnosticBatchNotificationParams { pub messages_len: usize, } +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DenoConfigurationData { + pub scope_uri: lsp::Uri, + pub workspace_root_scope_uri: Option, + pub deno_json: Option, + pub package_json: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DidRefreshDenoConfigurationTreeNotificationParams { + pub data: Vec, +} + +pub enum DidRefreshDenoConfigurationTreeNotification {} + +impl lsp::notification::Notification + for DidRefreshDenoConfigurationTreeNotification +{ + type Params = DidRefreshDenoConfigurationTreeNotificationParams; + const METHOD: &'static str = "deno/didRefreshDenoConfigurationTree"; +} + #[derive(Debug, Eq, Hash, PartialEq, Copy, Clone, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub enum DenoConfigurationChangeType { @@ -88,13 +110,15 @@ pub struct DidChangeDenoConfigurationNotificationParams { pub changes: Vec, } +// TODO(nayeemrmn): This is being replaced by +// `DidRefreshDenoConfigurationTreeNotification` for Deno > v2.0.0. Remove it +// soon. pub enum DidChangeDenoConfigurationNotification {} impl lsp::notification::Notification for DidChangeDenoConfigurationNotification { type Params = DidChangeDenoConfigurationNotificationParams; - const METHOD: &'static str = "deno/didChangeDenoConfiguration"; } @@ -102,7 +126,6 @@ pub enum DidUpgradeCheckNotification {} impl lsp::notification::Notification for DidUpgradeCheckNotification { type Params = DidUpgradeCheckNotificationParams; - const METHOD: &'static str = "deno/didUpgradeCheck"; } @@ -125,6 +148,5 @@ pub enum DiagnosticBatchNotification {} impl lsp::notification::Notification for DiagnosticBatchNotification { type Params = DiagnosticBatchNotificationParams; - const METHOD: &'static str = "deno/internalTestDiagnosticBatch"; } diff --git a/cli/lsp/mod.rs b/cli/lsp/mod.rs index 79aa4d8f074d14..afb949f68d0efa 100644 --- a/cli/lsp/mod.rs +++ b/cli/lsp/mod.rs @@ -56,9 +56,6 @@ pub async fn start() -> Result<(), AnyError> { LanguageServer::performance_request, ) .custom_method(lsp_custom::TASK_REQUEST, LanguageServer::task_definitions) - // TODO(nayeemrmn): Rename this to `deno/taskDefinitions` in vscode_deno and - // remove this alias. - .custom_method("deno/task", LanguageServer::task_definitions) .custom_method(testing::TEST_RUN_REQUEST, LanguageServer::test_run_request) .custom_method( testing::TEST_RUN_CANCEL_REQUEST, diff --git a/cli/lsp/npm.rs b/cli/lsp/npm.rs index 8bdeb7e7d8c517..2decfc342906f0 100644 --- a/cli/lsp/npm.rs +++ b/cli/lsp/npm.rs @@ -4,6 +4,7 @@ use dashmap::DashMap; use deno_core::anyhow::anyhow; use deno_core::error::AnyError; use deno_core::serde_json; +use deno_npm::npm_rc::NpmRc; use deno_semver::package::PackageNv; use deno_semver::Version; use serde::Deserialize; @@ -25,7 +26,10 @@ pub struct CliNpmSearchApi { impl CliNpmSearchApi { pub fn new(file_fetcher: Arc) -> Self { - let resolver = NpmFetchResolver::new(file_fetcher.clone()); + let resolver = NpmFetchResolver::new( + file_fetcher.clone(), + Arc::new(NpmRc::default().as_resolved(npm_registry_url()).unwrap()), + ); Self { file_fetcher, resolver, diff --git a/cli/lsp/parent_process_checker.rs b/cli/lsp/parent_process_checker.rs index e5b2b2f23115fd..b8a42cd1a4deb9 100644 --- a/cli/lsp/parent_process_checker.rs +++ b/cli/lsp/parent_process_checker.rs @@ -11,7 +11,7 @@ pub fn start(parent_process_id: u32) { std::thread::sleep(Duration::from_secs(10)); if !is_process_active(parent_process_id) { - std::process::exit(1); + deno_runtime::exit(1); } }); } diff --git a/cli/lsp/registries.rs b/cli/lsp/registries.rs index 5f7ce0082319f4..ade353e6834a99 100644 --- a/cli/lsp/registries.rs +++ b/cli/lsp/registries.rs @@ -482,6 +482,7 @@ impl ModuleRegistry { .fetch_with_options(FetchOptions { specifier: &specifier, permissions: FetchPermissionsOptionRef::AllowAll, + maybe_auth: None, maybe_accept: Some("application/vnd.deno.reg.v2+json, application/vnd.deno.reg.v1+json;q=0.9, application/json;q=0.8"), maybe_cache_setting: None, }) diff --git a/cli/lsp/repl.rs b/cli/lsp/repl.rs index fa5809045ed954..b4aaa8cd0d0594 100644 --- a/cli/lsp/repl.rs +++ b/cli/lsp/repl.rs @@ -263,7 +263,7 @@ impl ReplLanguageServer { } fn get_document_uri(&self) -> Uri { - uri_parse_unencoded(self.cwd_uri.join("$deno$repl.ts").unwrap().as_str()) + uri_parse_unencoded(self.cwd_uri.join("$deno$repl.mts").unwrap().as_str()) .unwrap() } } diff --git a/cli/lsp/resolver.rs b/cli/lsp/resolver.rs index 2844eb6f98adc8..c705511f30d260 100644 --- a/cli/lsp/resolver.rs +++ b/cli/lsp/resolver.rs @@ -2,27 +2,35 @@ use dashmap::DashMap; use deno_ast::MediaType; +use deno_cache_dir::npm::NpmCacheDir; use deno_cache_dir::HttpCache; +use deno_config::deno_json::JsxImportSourceConfig; use deno_config::workspace::PackageJsonDepResolution; use deno_config::workspace::WorkspaceResolver; +use deno_core::parking_lot::Mutex; use deno_core::url::Url; -use deno_graph::source::Resolver; use deno_graph::GraphImport; use deno_graph::ModuleSpecifier; +use deno_graph::Range; use deno_npm::NpmSystemInfo; +use deno_path_util::url_to_file_path; +use deno_resolver::cjs::IsCjsResolutionMode; +use deno_resolver::npm::NpmReqResolverOptions; +use deno_resolver::DenoResolverOptions; +use deno_resolver::NodeAndNpmReqResolver; use deno_runtime::deno_fs; use deno_runtime::deno_node::NodeResolver; use deno_runtime::deno_node::PackageJson; -use deno_runtime::fs_util::specifier_to_file_path; +use deno_runtime::deno_node::PackageJsonResolver; use deno_semver::jsr::JsrPackageReqReference; use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; use indexmap::IndexMap; use node_resolver::errors::ClosestPkgJsonError; -use node_resolver::NodeResolution; -use node_resolver::NodeResolutionMode; -use node_resolver::NpmResolver; +use node_resolver::InNpmPackageChecker; +use node_resolver::NodeResolutionKind; +use node_resolver::ResolutionMode; use std::borrow::Cow; use std::collections::BTreeMap; use std::collections::BTreeSet; @@ -36,46 +44,68 @@ use crate::args::create_default_npmrc; use crate::args::CacheSetting; use crate::args::CliLockfile; use crate::args::NpmInstallDepsProvider; +use crate::cache::DenoCacheEnvFsAdapter; +use crate::factory::Deferred; +use crate::graph_util::to_node_resolution_kind; +use crate::graph_util::to_node_resolution_mode; use crate::graph_util::CliJsrUrlProvider; use crate::http_util::HttpClientProvider; use crate::lsp::config::Config; use crate::lsp::config::ConfigData; use crate::lsp::logging::lsp_warn; use crate::npm::create_cli_npm_resolver_for_lsp; +use crate::npm::CliByonmNpmResolverCreateOptions; +use crate::npm::CliManagedInNpmPkgCheckerCreateOptions; +use crate::npm::CliManagedNpmResolverCreateOptions; use crate::npm::CliNpmResolver; -use crate::npm::CliNpmResolverByonmCreateOptions; use crate::npm::CliNpmResolverCreateOptions; -use crate::npm::CliNpmResolverManagedCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; +use crate::npm::CreateInNpmPkgCheckerOptions; use crate::npm::ManagedCliNpmResolver; -use crate::resolver::CjsResolutionStore; -use crate::resolver::CliGraphResolver; -use crate::resolver::CliGraphResolverOptions; -use crate::resolver::CliNodeResolver; +use crate::resolver::CliDenoResolver; +use crate::resolver::CliDenoResolverFs; +use crate::resolver::CliNpmReqResolver; +use crate::resolver::CliResolver; +use crate::resolver::CliResolverOptions; +use crate::resolver::IsCjsResolver; use crate::resolver::WorkerCliNpmGraphResolver; +use crate::tsc::into_specifier_and_media_type; use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; #[derive(Debug, Clone)] struct LspScopeResolver { - graph_resolver: Arc, + resolver: Arc, + in_npm_pkg_checker: Arc, + is_cjs_resolver: Arc, jsr_resolver: Option>, npm_resolver: Option>, - node_resolver: Option>, + node_resolver: Option>, + npm_pkg_req_resolver: Option>, + pkg_json_resolver: Arc, redirect_resolver: Option>, graph_imports: Arc>, + dep_info: Arc>>, + package_json_deps_by_resolution: Arc>, config_data: Option>, } impl Default for LspScopeResolver { fn default() -> Self { + let factory = ResolverFactory::new(None); Self { - graph_resolver: create_graph_resolver(None, None, None), + resolver: factory.cli_resolver().clone(), + in_npm_pkg_checker: factory.in_npm_pkg_checker().clone(), + is_cjs_resolver: factory.is_cjs_resolver().clone(), jsr_resolver: None, npm_resolver: None, node_resolver: None, + npm_pkg_req_resolver: None, + pkg_json_resolver: factory.pkg_json_resolver().clone(), redirect_resolver: None, graph_imports: Default::default(), + dep_info: Default::default(), + package_json_deps_by_resolution: Default::default(), config_data: None, } } @@ -87,22 +117,16 @@ impl LspScopeResolver { cache: &LspCache, http_client_provider: Option<&Arc>, ) -> Self { - let mut npm_resolver = None; - let mut node_resolver = None; - if let Some(http_client) = http_client_provider { - npm_resolver = create_npm_resolver( - config_data.map(|d| d.as_ref()), - cache, - http_client, - ) - .await; - node_resolver = create_node_resolver(npm_resolver.as_ref()); + let mut factory = ResolverFactory::new(config_data); + if let Some(http_client_provider) = http_client_provider { + factory.init_npm_resolver(http_client_provider, cache).await; } - let graph_resolver = create_graph_resolver( - config_data.map(|d| d.as_ref()), - npm_resolver.as_ref(), - node_resolver.as_ref(), - ); + let in_npm_pkg_checker = factory.in_npm_pkg_checker().clone(); + let npm_resolver = factory.npm_resolver().cloned(); + let node_resolver = factory.node_resolver().cloned(); + let npm_pkg_req_resolver = factory.npm_pkg_req_resolver().cloned(); + let cli_resolver = factory.cli_resolver().clone(); + let pkg_json_resolver = factory.pkg_json_resolver().clone(); let jsr_resolver = Some(Arc::new(JsrCacheResolver::new( cache.for_specifier(config_data.map(|d| d.scope.as_ref())), config_data.map(|d| d.as_ref()), @@ -111,7 +135,9 @@ impl LspScopeResolver { cache.for_specifier(config_data.map(|d| d.scope.as_ref())), config_data.and_then(|d| d.lockfile.clone()), ))); - let npm_graph_resolver = graph_resolver.create_graph_npm_resolver(); + let npm_graph_resolver = cli_resolver.create_graph_npm_resolver(); + let maybe_jsx_import_source_config = + config_data.and_then(|d| d.maybe_jsx_import_source_config()); let graph_imports = config_data .and_then(|d| d.member_dir.workspace.to_compiler_option_types().ok()) .map(|imports| { @@ -119,11 +145,18 @@ impl LspScopeResolver { imports .into_iter() .map(|(referrer, imports)| { + let resolver = SingleReferrerGraphResolver { + valid_referrer: &referrer, + module_resolution_mode: ResolutionMode::Import, + cli_resolver: &cli_resolver, + jsx_import_source_config: maybe_jsx_import_source_config + .as_ref(), + }; let graph_import = GraphImport::new( &referrer, imports, &CliJsrUrlProvider, - Some(graph_resolver.as_ref()), + Some(&resolver), Some(&npm_graph_resolver), ); (referrer, graph_import) @@ -132,33 +165,83 @@ impl LspScopeResolver { ) }) .unwrap_or_default(); + let package_json_deps_by_resolution = (|| { + let npm_pkg_req_resolver = npm_pkg_req_resolver.as_ref()?; + let package_json = config_data?.maybe_pkg_json()?; + let referrer = package_json.specifier(); + let dependencies = package_json.dependencies.as_ref()?; + let result = dependencies + .iter() + .flat_map(|(name, _)| { + let req_ref = + NpmPackageReqReference::from_str(&format!("npm:{name}")).ok()?; + let specifier = into_specifier_and_media_type(Some( + npm_pkg_req_resolver + .resolve_req_reference( + &req_ref, + &referrer, + // todo(dsherret): this is wrong because it doesn't consider CJS referrers + ResolutionMode::Import, + NodeResolutionKind::Types, + ) + .or_else(|_| { + npm_pkg_req_resolver.resolve_req_reference( + &req_ref, + &referrer, + // todo(dsherret): this is wrong because it doesn't consider CJS referrers + ResolutionMode::Import, + NodeResolutionKind::Execution, + ) + }) + .ok()?, + )) + .0; + Some((specifier, name.clone())) + }) + .collect(); + Some(result) + })(); + let package_json_deps_by_resolution = + Arc::new(package_json_deps_by_resolution.unwrap_or_default()); Self { - graph_resolver, + resolver: cli_resolver, + in_npm_pkg_checker, + is_cjs_resolver: factory.is_cjs_resolver().clone(), jsr_resolver, + npm_pkg_req_resolver, npm_resolver, node_resolver, + pkg_json_resolver, redirect_resolver, graph_imports, + dep_info: Default::default(), + package_json_deps_by_resolution, config_data: config_data.cloned(), } } fn snapshot(&self) -> Arc { + let mut factory = ResolverFactory::new(self.config_data.as_ref()); let npm_resolver = self.npm_resolver.as_ref().map(|r| r.clone_snapshotted()); - let node_resolver = create_node_resolver(npm_resolver.as_ref()); - let graph_resolver = create_graph_resolver( - self.config_data.as_deref(), - npm_resolver.as_ref(), - node_resolver.as_ref(), - ); + if let Some(npm_resolver) = &npm_resolver { + factory.set_npm_resolver(npm_resolver.clone()); + } Arc::new(Self { - graph_resolver, + resolver: factory.cli_resolver().clone(), + in_npm_pkg_checker: factory.in_npm_pkg_checker().clone(), + is_cjs_resolver: factory.is_cjs_resolver().clone(), jsr_resolver: self.jsr_resolver.clone(), - npm_resolver, - node_resolver, + npm_pkg_req_resolver: factory.npm_pkg_req_resolver().cloned(), + npm_resolver: factory.npm_resolver().cloned(), + node_resolver: factory.node_resolver().cloned(), redirect_resolver: self.redirect_resolver.clone(), + pkg_json_resolver: factory.pkg_json_resolver().clone(), graph_imports: self.graph_imports.clone(), + dep_info: self.dep_info.clone(), + package_json_deps_by_resolution: self + .package_json_deps_by_resolution + .clone(), config_data: self.config_data.clone(), }) } @@ -222,19 +305,24 @@ impl LspResolver { } } - pub async fn set_npm_reqs( + pub async fn set_dep_info_by_scope( &self, - reqs: &BTreeMap, BTreeSet>, + dep_info_by_scope: &Arc< + BTreeMap, Arc>, + >, ) { for (scope, resolver) in [(None, &self.unscoped)] .into_iter() .chain(self.by_scope.iter().map(|(s, r)| (Some(s), r))) { + let dep_info = dep_info_by_scope.get(&scope.cloned()); + if let Some(dep_info) = dep_info { + *resolver.dep_info.lock() = dep_info.clone(); + } if let Some(npm_resolver) = resolver.npm_resolver.as_ref() { if let Some(npm_resolver) = npm_resolver.as_managed() { - let reqs = reqs - .get(&scope.cloned()) - .map(|reqs| reqs.iter().cloned().collect::>()) + let reqs = dep_info + .map(|i| i.npm_reqs.iter().cloned().collect::>()) .unwrap_or_default(); if let Err(err) = npm_resolver.set_package_reqs(&reqs).await { lsp_warn!("Could not set npm package requirements: {:#}", err); @@ -244,12 +332,12 @@ impl LspResolver { } } - pub fn as_graph_resolver( + pub fn as_cli_resolver( &self, file_referrer: Option<&ModuleSpecifier>, - ) -> &dyn Resolver { + ) -> &CliResolver { let resolver = self.get_scope_resolver(file_referrer); - resolver.graph_resolver.as_ref() + resolver.resolver.as_ref() } pub fn create_graph_npm_resolver( @@ -257,7 +345,31 @@ impl LspResolver { file_referrer: Option<&ModuleSpecifier>, ) -> WorkerCliNpmGraphResolver { let resolver = self.get_scope_resolver(file_referrer); - resolver.graph_resolver.create_graph_npm_resolver() + resolver.resolver.create_graph_npm_resolver() + } + + pub fn as_is_cjs_resolver( + &self, + file_referrer: Option<&ModuleSpecifier>, + ) -> &IsCjsResolver { + let resolver = self.get_scope_resolver(file_referrer); + resolver.is_cjs_resolver.as_ref() + } + + pub fn as_config_data( + &self, + file_referrer: Option<&ModuleSpecifier>, + ) -> Option<&Arc> { + let resolver = self.get_scope_resolver(file_referrer); + resolver.config_data.as_ref() + } + + pub fn in_npm_pkg_checker( + &self, + file_referrer: Option<&ModuleSpecifier>, + ) -> &Arc { + let resolver = self.get_scope_resolver(file_referrer); + &resolver.in_npm_pkg_checker } pub fn maybe_managed_npm_resolver( @@ -323,15 +435,46 @@ impl LspResolver { &self, req_ref: &NpmPackageReqReference, referrer: &ModuleSpecifier, + resolution_mode: ResolutionMode, file_referrer: Option<&ModuleSpecifier>, ) -> Option<(ModuleSpecifier, MediaType)> { let resolver = self.get_scope_resolver(file_referrer); - let node_resolver = resolver.node_resolver.as_ref()?; - Some(NodeResolution::into_specifier_and_media_type( - node_resolver - .resolve_req_reference(req_ref, referrer, NodeResolutionMode::Types) - .ok(), - )) + let npm_pkg_req_resolver = resolver.npm_pkg_req_resolver.as_ref()?; + Some(into_specifier_and_media_type(Some( + npm_pkg_req_resolver + .resolve_req_reference( + req_ref, + referrer, + resolution_mode, + NodeResolutionKind::Types, + ) + .ok()?, + ))) + } + + pub fn file_url_to_package_json_dep( + &self, + specifier: &ModuleSpecifier, + file_referrer: Option<&ModuleSpecifier>, + ) -> Option { + let resolver = self.get_scope_resolver(file_referrer); + resolver + .package_json_deps_by_resolution + .get(specifier) + .cloned() + } + + pub fn deno_types_to_code_resolution( + &self, + specifier: &ModuleSpecifier, + file_referrer: Option<&ModuleSpecifier>, + ) -> Option { + let resolver = self.get_scope_resolver(file_referrer); + let dep_info = resolver.dep_info.lock().clone(); + dep_info + .deno_types_to_code_resolutions + .get(specifier) + .cloned() } pub fn in_node_modules(&self, specifier: &ModuleSpecifier) -> bool { @@ -345,14 +488,10 @@ impl LspResolver { .contains("/node_modules/") } - let global_npm_resolver = self - .get_scope_resolver(Some(specifier)) - .npm_resolver - .as_ref() - .and_then(|npm_resolver| npm_resolver.as_managed()) - .filter(|r| r.root_node_modules_path().is_none()); - if let Some(npm_resolver) = &global_npm_resolver { - if npm_resolver.in_npm_package(specifier) { + if let Some(node_resolver) = + &self.get_scope_resolver(Some(specifier)).node_resolver + { + if node_resolver.in_npm_package(specifier) { return true; } } @@ -360,16 +499,27 @@ impl LspResolver { has_node_modules_dir(specifier) } - pub fn node_media_type( + pub fn is_bare_package_json_dep( &self, - specifier: &ModuleSpecifier, - ) -> Option { - let resolver = self.get_scope_resolver(Some(specifier)); - let node_resolver = resolver.node_resolver.as_ref()?; - let resolution = node_resolver - .url_to_node_resolution(specifier.clone()) - .ok()?; - Some(NodeResolution::into_specifier_and_media_type(Some(resolution)).1) + specifier_text: &str, + referrer: &ModuleSpecifier, + resolution_mode: ResolutionMode, + ) -> bool { + let resolver = self.get_scope_resolver(Some(referrer)); + let Some(npm_pkg_req_resolver) = resolver.npm_pkg_req_resolver.as_ref() + else { + return false; + }; + npm_pkg_req_resolver + .resolve_if_for_npm_pkg( + specifier_text, + referrer, + resolution_mode, + NodeResolutionKind::Types, + ) + .ok() + .flatten() + .is_some() } pub fn get_closest_package_json( @@ -377,10 +527,9 @@ impl LspResolver { referrer: &ModuleSpecifier, ) -> Result>, ClosestPkgJsonError> { let resolver = self.get_scope_resolver(Some(referrer)); - let Some(node_resolver) = resolver.node_resolver.as_ref() else { - return Ok(None); - }; - node_resolver.get_closest_package_json(referrer) + resolver + .pkg_json_resolver + .get_closest_package_json(referrer) } pub fn resolve_redirects( @@ -432,113 +581,231 @@ impl LspResolver { } } -async fn create_npm_resolver( - config_data: Option<&ConfigData>, - cache: &LspCache, - http_client_provider: &Arc, -) -> Option> { - let enable_byonm = config_data.map(|d| d.byonm).unwrap_or(false); - let options = if enable_byonm { - CliNpmResolverCreateOptions::Byonm(CliNpmResolverByonmCreateOptions { - fs: Arc::new(deno_fs::RealFs), - root_node_modules_dir: config_data.and_then(|config_data| { - config_data.node_modules_dir.clone().or_else(|| { - specifier_to_file_path(&config_data.scope) - .ok() - .map(|p| p.join("node_modules/")) - }) - }), - }) - } else { - CliNpmResolverCreateOptions::Managed(CliNpmResolverManagedCreateOptions { - http_client_provider: http_client_provider.clone(), - snapshot: match config_data.and_then(|d| d.lockfile.as_ref()) { - Some(lockfile) => { - CliNpmResolverManagedSnapshotOption::ResolveFromLockfile( - lockfile.clone(), - ) - } - None => CliNpmResolverManagedSnapshotOption::Specified(None), - }, - // Don't provide the lockfile. We don't want these resolvers - // updating it. Only the cache request should update the lockfile. - maybe_lockfile: None, - fs: Arc::new(deno_fs::RealFs), - npm_global_cache_dir: cache.deno_dir().npm_folder_path(), - // Use an "only" cache setting in order to make the - // user do an explicit "cache" command and prevent - // the cache from being filled with lots of packages while - // the user is typing. - cache_setting: CacheSetting::Only, - text_only_progress_bar: ProgressBar::new(ProgressBarStyle::TextOnly), - maybe_node_modules_path: config_data - .and_then(|d| d.node_modules_dir.clone()), - // only used for top level install, so we can ignore this - npm_install_deps_provider: Arc::new(NpmInstallDepsProvider::empty()), - npmrc: config_data - .and_then(|d| d.npmrc.clone()) - .unwrap_or_else(create_default_npmrc), - npm_system_info: NpmSystemInfo::default(), - lifecycle_scripts: Default::default(), - }) - }; - Some(create_cli_npm_resolver_for_lsp(options).await) +#[derive(Debug, Default, Clone)] +pub struct ScopeDepInfo { + pub deno_types_to_code_resolutions: HashMap, + pub npm_reqs: BTreeSet, + pub has_node_specifier: bool, } -fn create_node_resolver( - npm_resolver: Option<&Arc>, -) -> Option> { - use once_cell::sync::Lazy; - - // it's not ideal to share this across all scopes and to - // never clear it, but it's fine for the time being - static CJS_RESOLUTIONS: Lazy> = - Lazy::new(Default::default); - - let npm_resolver = npm_resolver?; - let fs = Arc::new(deno_fs::RealFs); - let node_resolver_inner = Arc::new(NodeResolver::new( - deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()), - npm_resolver.clone().into_npm_resolver(), - )); - Some(Arc::new(CliNodeResolver::new( - CJS_RESOLUTIONS.clone(), - fs, - node_resolver_inner, - npm_resolver.clone(), - ))) +#[derive(Default)] +struct ResolverFactoryServices { + cli_resolver: Deferred>, + in_npm_pkg_checker: Deferred>, + is_cjs_resolver: Deferred>, + node_resolver: Deferred>>, + npm_pkg_req_resolver: Deferred>>, + npm_resolver: Option>, } -fn create_graph_resolver( - config_data: Option<&ConfigData>, - npm_resolver: Option<&Arc>, - node_resolver: Option<&Arc>, -) -> Arc { - let workspace = config_data.map(|d| &d.member_dir.workspace); - Arc::new(CliGraphResolver::new(CliGraphResolverOptions { - node_resolver: node_resolver.cloned(), - npm_resolver: npm_resolver.cloned(), - workspace_resolver: config_data.map(|d| d.resolver.clone()).unwrap_or_else( - || { - Arc::new(WorkspaceResolver::new_raw( - // this is fine because this is only used before initialization - Arc::new(ModuleSpecifier::parse("file:///").unwrap()), - None, - Vec::new(), - Vec::new(), - PackageJsonDepResolution::Disabled, - )) - }, - ), - maybe_jsx_import_source_config: workspace.and_then(|workspace| { - workspace.to_maybe_jsx_import_source_config().ok().flatten() - }), - maybe_vendor_dir: config_data.and_then(|d| d.vendor_dir.as_ref()), - bare_node_builtins_enabled: workspace - .is_some_and(|workspace| workspace.has_unstable("bare-node-builtins")), - sloppy_imports_resolver: config_data - .and_then(|d| d.sloppy_imports_resolver.clone()), - })) +struct ResolverFactory<'a> { + config_data: Option<&'a Arc>, + fs: Arc, + pkg_json_resolver: Arc, + services: ResolverFactoryServices, +} + +impl<'a> ResolverFactory<'a> { + pub fn new(config_data: Option<&'a Arc>) -> Self { + let fs = Arc::new(deno_fs::RealFs); + let pkg_json_resolver = Arc::new(PackageJsonResolver::new( + deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()), + )); + Self { + config_data, + fs, + pkg_json_resolver, + services: Default::default(), + } + } + + async fn init_npm_resolver( + &mut self, + http_client_provider: &Arc, + cache: &LspCache, + ) { + let enable_byonm = self.config_data.map(|d| d.byonm).unwrap_or(false); + let options = if enable_byonm { + CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions { + fs: CliDenoResolverFs(Arc::new(deno_fs::RealFs)), + pkg_json_resolver: self.pkg_json_resolver.clone(), + root_node_modules_dir: self.config_data.and_then(|config_data| { + config_data.node_modules_dir.clone().or_else(|| { + url_to_file_path(&config_data.scope) + .ok() + .map(|p| p.join("node_modules/")) + }) + }), + }) + } else { + let npmrc = self + .config_data + .and_then(|d| d.npmrc.clone()) + .unwrap_or_else(create_default_npmrc); + let npm_cache_dir = Arc::new(NpmCacheDir::new( + &DenoCacheEnvFsAdapter(self.fs.as_ref()), + cache.deno_dir().npm_folder_path(), + npmrc.get_all_known_registries_urls(), + )); + CliNpmResolverCreateOptions::Managed(CliManagedNpmResolverCreateOptions { + http_client_provider: http_client_provider.clone(), + snapshot: match self.config_data.and_then(|d| d.lockfile.as_ref()) { + Some(lockfile) => { + CliNpmResolverManagedSnapshotOption::ResolveFromLockfile( + lockfile.clone(), + ) + } + None => CliNpmResolverManagedSnapshotOption::Specified(None), + }, + // Don't provide the lockfile. We don't want these resolvers + // updating it. Only the cache request should update the lockfile. + maybe_lockfile: None, + fs: Arc::new(deno_fs::RealFs), + npm_cache_dir, + // Use an "only" cache setting in order to make the + // user do an explicit "cache" command and prevent + // the cache from being filled with lots of packages while + // the user is typing. + cache_setting: CacheSetting::Only, + text_only_progress_bar: ProgressBar::new(ProgressBarStyle::TextOnly), + maybe_node_modules_path: self + .config_data + .and_then(|d| d.node_modules_dir.clone()), + // only used for top level install, so we can ignore this + npm_install_deps_provider: Arc::new(NpmInstallDepsProvider::empty()), + npmrc, + npm_system_info: NpmSystemInfo::default(), + lifecycle_scripts: Default::default(), + }) + }; + self.set_npm_resolver(create_cli_npm_resolver_for_lsp(options).await); + } + + pub fn set_npm_resolver(&mut self, npm_resolver: Arc) { + self.services.npm_resolver = Some(npm_resolver); + } + + pub fn npm_resolver(&self) -> Option<&Arc> { + self.services.npm_resolver.as_ref() + } + + pub fn cli_resolver(&self) -> &Arc { + self.services.cli_resolver.get_or_init(|| { + let npm_req_resolver = self.npm_pkg_req_resolver().cloned(); + let deno_resolver = Arc::new(CliDenoResolver::new(DenoResolverOptions { + in_npm_pkg_checker: self.in_npm_pkg_checker().clone(), + node_and_req_resolver: match (self.node_resolver(), npm_req_resolver) { + (Some(node_resolver), Some(npm_req_resolver)) => { + Some(NodeAndNpmReqResolver { + node_resolver: node_resolver.clone(), + npm_req_resolver, + }) + } + _ => None, + }, + sloppy_imports_resolver: self + .config_data + .and_then(|d| d.sloppy_imports_resolver.clone()), + workspace_resolver: self + .config_data + .map(|d| d.resolver.clone()) + .unwrap_or_else(|| { + Arc::new(WorkspaceResolver::new_raw( + // this is fine because this is only used before initialization + Arc::new(ModuleSpecifier::parse("file:///").unwrap()), + None, + Vec::new(), + Vec::new(), + PackageJsonDepResolution::Disabled, + )) + }), + is_byonm: self.config_data.map(|d| d.byonm).unwrap_or(false), + maybe_vendor_dir: self.config_data.and_then(|d| d.vendor_dir.as_ref()), + })); + Arc::new(CliResolver::new(CliResolverOptions { + deno_resolver, + npm_resolver: self.npm_resolver().cloned(), + bare_node_builtins_enabled: self + .config_data + .is_some_and(|d| d.unstable.contains("bare-node-builtins")), + })) + }) + } + + pub fn pkg_json_resolver(&self) -> &Arc { + &self.pkg_json_resolver + } + + pub fn in_npm_pkg_checker(&self) -> &Arc { + self.services.in_npm_pkg_checker.get_or_init(|| { + crate::npm::create_in_npm_pkg_checker( + match self.services.npm_resolver.as_ref().map(|r| r.as_inner()) { + Some(crate::npm::InnerCliNpmResolverRef::Byonm(_)) | None => { + CreateInNpmPkgCheckerOptions::Byonm + } + Some(crate::npm::InnerCliNpmResolverRef::Managed(m)) => { + CreateInNpmPkgCheckerOptions::Managed( + CliManagedInNpmPkgCheckerCreateOptions { + root_cache_dir_url: m.global_cache_root_url(), + maybe_node_modules_path: m.maybe_node_modules_path(), + }, + ) + } + }, + ) + }) + } + + pub fn is_cjs_resolver(&self) -> &Arc { + self.services.is_cjs_resolver.get_or_init(|| { + Arc::new(IsCjsResolver::new( + self.in_npm_pkg_checker().clone(), + self.pkg_json_resolver().clone(), + if self + .config_data + .is_some_and(|d| d.unstable.contains("detect-cjs")) + { + IsCjsResolutionMode::ImplicitTypeCommonJs + } else { + IsCjsResolutionMode::ExplicitTypeCommonJs + }, + )) + }) + } + + pub fn node_resolver(&self) -> Option<&Arc> { + self + .services + .node_resolver + .get_or_init(|| { + let npm_resolver = self.services.npm_resolver.as_ref()?; + Some(Arc::new(NodeResolver::new( + deno_runtime::deno_node::DenoFsNodeResolverEnv::new(self.fs.clone()), + self.in_npm_pkg_checker().clone(), + npm_resolver.clone().into_npm_pkg_folder_resolver(), + self.pkg_json_resolver.clone(), + ))) + }) + .as_ref() + } + + pub fn npm_pkg_req_resolver(&self) -> Option<&Arc> { + self + .services + .npm_pkg_req_resolver + .get_or_init(|| { + let node_resolver = self.node_resolver()?; + let npm_resolver = self.npm_resolver()?; + Some(Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions { + byonm_resolver: (npm_resolver.clone()).into_maybe_byonm(), + fs: CliDenoResolverFs(self.fs.clone()), + in_npm_pkg_checker: self.in_npm_pkg_checker().clone(), + node_resolver: node_resolver.clone(), + npm_req_resolver: npm_resolver.clone().into_npm_req_resolver(), + }))) + }) + .as_ref() + } } #[derive(Debug, Eq, PartialEq)] @@ -565,6 +832,56 @@ impl std::fmt::Debug for RedirectResolver { } } +#[derive(Debug)] +pub struct SingleReferrerGraphResolver<'a> { + pub valid_referrer: &'a ModuleSpecifier, + pub module_resolution_mode: ResolutionMode, + pub cli_resolver: &'a CliResolver, + pub jsx_import_source_config: Option<&'a JsxImportSourceConfig>, +} + +impl<'a> deno_graph::source::Resolver for SingleReferrerGraphResolver<'a> { + fn default_jsx_import_source(&self) -> Option { + self + .jsx_import_source_config + .and_then(|c| c.default_specifier.clone()) + } + + fn default_jsx_import_source_types(&self) -> Option { + self + .jsx_import_source_config + .and_then(|c| c.default_types_specifier.clone()) + } + + fn jsx_import_source_module(&self) -> &str { + self + .jsx_import_source_config + .map(|c| c.module.as_str()) + .unwrap_or(deno_graph::source::DEFAULT_JSX_IMPORT_SOURCE_MODULE) + } + + fn resolve( + &self, + specifier_text: &str, + referrer_range: &Range, + resolution_kind: deno_graph::source::ResolutionKind, + ) -> Result { + // this resolver assumes it will only be used with a single referrer + // with the provided referrer kind + debug_assert_eq!(referrer_range.specifier, *self.valid_referrer); + self.cli_resolver.resolve( + specifier_text, + &referrer_range.specifier, + referrer_range.range.start, + referrer_range + .resolution_mode + .map(to_node_resolution_mode) + .unwrap_or(self.module_resolution_mode), + to_node_resolution_kind(resolution_kind), + ) + } +} + impl RedirectResolver { fn new( cache: Arc, diff --git a/cli/lsp/testing/collectors.rs b/cli/lsp/testing/collectors.rs index 2f2ddb8773ba77..2dd7ec0d96d342 100644 --- a/cli/lsp/testing/collectors.rs +++ b/cli/lsp/testing/collectors.rs @@ -650,7 +650,7 @@ pub mod tests { .unwrap(); let text_info = parsed_module.text_info_lazy().clone(); let mut collector = TestCollector::new(specifier, text_info); - parsed_module.module().visit_with(&mut collector); + parsed_module.program().visit_with(&mut collector); collector.take() } diff --git a/cli/lsp/tsc.rs b/cli/lsp/tsc.rs index fe3708d3cb5ad3..a8e2b91e775224 100644 --- a/cli/lsp/tsc.rs +++ b/cli/lsp/tsc.rs @@ -34,6 +34,7 @@ use crate::util::path::relative_specifier; use crate::util::path::to_percent_decoded_str; use crate::util::result::InfallibleResultExt; use crate::util::v8::convert; +use crate::worker::create_isolate_create_params; use deno_core::convert::Smi; use deno_core::convert::ToV8; use deno_core::error::StdAnyError; @@ -62,13 +63,14 @@ use deno_core::ModuleSpecifier; use deno_core::OpState; use deno_core::PollEventLoopOptions; use deno_core::RuntimeOptions; -use deno_runtime::fs_util::specifier_to_file_path; +use deno_path_util::url_to_file_path; use deno_runtime::inspector_server::InspectorServer; use deno_runtime::tokio_util::create_basic_runtime; use indexmap::IndexMap; use indexmap::IndexSet; use lazy_regex::lazy_regex; use log::error; +use node_resolver::ResolutionMode; use once_cell::sync::Lazy; use regex::Captures; use regex::Regex; @@ -236,7 +238,7 @@ pub struct TsServer { performance: Arc, sender: mpsc::UnboundedSender, receiver: Mutex>>, - specifier_map: Arc, + pub specifier_map: Arc, inspector_server: Mutex>>, pending_change: Mutex>, } @@ -882,20 +884,22 @@ impl TsServer { options: GetCompletionsAtPositionOptions, format_code_settings: FormatCodeSettings, scope: Option, - ) -> Option { + ) -> Result, AnyError> { let req = TscRequest::GetCompletionsAtPosition(Box::new(( self.specifier_map.denormalize(&specifier), position, options, format_code_settings, ))); - match self.request(snapshot, req, scope).await { - Ok(maybe_info) => maybe_info, - Err(err) => { - log::error!("Unable to get completion info from TypeScript: {:#}", err); - None - } - } + self + .request::>(snapshot, req, scope) + .await + .map(|mut info| { + if let Some(info) = &mut info { + info.normalize(&self.specifier_map); + } + info + }) } pub async fn get_completion_details( @@ -2182,6 +2186,50 @@ impl NavigateToItem { } } +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct InlayHintDisplayPart { + pub text: String, + pub span: Option, + pub file: Option, +} + +impl InlayHintDisplayPart { + pub fn to_lsp( + &self, + language_server: &language_server::Inner, + ) -> lsp::InlayHintLabelPart { + let location = self.file.as_ref().map(|f| { + let specifier = + resolve_url(f).unwrap_or_else(|_| INVALID_SPECIFIER.clone()); + let file_referrer = + language_server.documents.get_file_referrer(&specifier); + let uri = language_server + .url_map + .specifier_to_uri(&specifier, file_referrer.as_deref()) + .unwrap_or_else(|_| INVALID_URI.clone()); + let range = self + .span + .as_ref() + .and_then(|s| { + let asset_or_doc = + language_server.get_asset_or_document(&specifier).ok()?; + Some(s.to_range(asset_or_doc.line_index())) + }) + .unwrap_or_else(|| { + lsp::Range::new(lsp::Position::new(0, 0), lsp::Position::new(0, 0)) + }); + lsp::Location { uri, range } + }); + lsp::InlayHintLabelPart { + value: self.text.clone(), + tooltip: None, + location, + command: None, + } + } +} + #[derive(Debug, Clone, Deserialize)] pub enum InlayHintKind { Type, @@ -2203,6 +2251,7 @@ impl InlayHintKind { #[serde(rename_all = "camelCase")] pub struct InlayHint { pub text: String, + pub display_parts: Option>, pub position: u32, pub kind: InlayHintKind, pub whitespace_before: Option, @@ -2210,10 +2259,23 @@ pub struct InlayHint { } impl InlayHint { - pub fn to_lsp(&self, line_index: Arc) -> lsp::InlayHint { + pub fn to_lsp( + &self, + line_index: Arc, + language_server: &language_server::Inner, + ) -> lsp::InlayHint { lsp::InlayHint { position: line_index.position_tsc(self.position.into()), - label: lsp::InlayHintLabel::String(self.text.clone()), + label: if let Some(display_parts) = &self.display_parts { + lsp::InlayHintLabel::LabelParts( + display_parts + .iter() + .map(|p| p.to_lsp(language_server)) + .collect(), + ) + } else { + lsp::InlayHintLabel::String(self.text.clone()) + }, kind: self.kind.to_lsp(), padding_left: self.whitespace_before, padding_right: self.whitespace_after, @@ -3191,7 +3253,7 @@ impl CallHierarchyItem { let use_file_name = self.is_source_file_item(); let maybe_file_path = if uri.scheme().is_some_and(|s| s.as_str() == "file") { - specifier_to_file_path(&uri_to_url(&uri)).ok() + url_to_file_path(&uri_to_url(&uri)).ok() } else { None }; @@ -3355,9 +3417,18 @@ fn parse_code_actions( additional_text_edits.extend(change.text_changes.iter().map(|tc| { let mut text_edit = tc.as_text_edit(asset_or_doc.line_index()); if let Some(specifier_rewrite) = &data.specifier_rewrite { - text_edit.new_text = text_edit - .new_text - .replace(&specifier_rewrite.0, &specifier_rewrite.1); + text_edit.new_text = text_edit.new_text.replace( + &specifier_rewrite.old_specifier, + &specifier_rewrite.new_specifier, + ); + if let Some(deno_types_specifier) = + &specifier_rewrite.new_deno_types_specifier + { + text_edit.new_text = format!( + "// @deno-types=\"{}\"\n{}", + deno_types_specifier, &text_edit.new_text + ); + } } text_edit })); @@ -3516,17 +3587,23 @@ impl CompletionEntryDetails { let mut text_edit = original_item.text_edit.clone(); if let Some(specifier_rewrite) = &data.specifier_rewrite { if let Some(text_edit) = &mut text_edit { - match text_edit { - lsp::CompletionTextEdit::Edit(text_edit) => { - text_edit.new_text = text_edit - .new_text - .replace(&specifier_rewrite.0, &specifier_rewrite.1); - } + let new_text = match text_edit { + lsp::CompletionTextEdit::Edit(text_edit) => &mut text_edit.new_text, lsp::CompletionTextEdit::InsertAndReplace(insert_replace_edit) => { - insert_replace_edit.new_text = insert_replace_edit - .new_text - .replace(&specifier_rewrite.0, &specifier_rewrite.1); + &mut insert_replace_edit.new_text } + }; + *new_text = new_text.replace( + &specifier_rewrite.old_specifier, + &specifier_rewrite.new_specifier, + ); + if let Some(deno_types_specifier) = + &specifier_rewrite.new_deno_types_specifier + { + *new_text = format!( + "// @deno-types=\"{}\"\n{}", + deno_types_specifier, new_text + ); } } } @@ -3584,6 +3661,12 @@ pub struct CompletionInfo { } impl CompletionInfo { + fn normalize(&mut self, specifier_map: &TscSpecifierMap) { + for entry in &mut self.entries { + entry.normalize(specifier_map); + } + } + pub fn as_completion_response( &self, line_index: Arc, @@ -3625,6 +3708,13 @@ impl CompletionInfo { } } +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionSpecifierRewrite { + old_specifier: String, + new_specifier: String, + new_deno_types_specifier: Option, +} + #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct CompletionItemData { @@ -3637,7 +3727,7 @@ pub struct CompletionItemData { /// be rewritten by replacing the first string with the second. Intended for /// auto-import specifiers to be reverse-import-mapped. #[serde(skip_serializing_if = "Option::is_none")] - pub specifier_rewrite: Option<(String, String)>, + pub specifier_rewrite: Option, #[serde(skip_serializing_if = "Option::is_none")] pub data: Option, pub use_code_snippet: bool, @@ -3645,11 +3735,17 @@ pub struct CompletionItemData { #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] -struct CompletionEntryDataImport { +struct CompletionEntryDataAutoImport { module_specifier: String, file_name: String, } +#[derive(Debug)] +pub struct CompletionNormalizedAutoImportData { + raw: CompletionEntryDataAutoImport, + normalized: ModuleSpecifier, +} + #[derive(Debug, Default, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct CompletionEntry { @@ -3682,9 +3778,28 @@ pub struct CompletionEntry { is_import_statement_completion: Option, #[serde(skip_serializing_if = "Option::is_none")] data: Option, + /// This is not from tsc, we add it for convenience during normalization. + /// Represents `self.data.file_name`, but normalized. + #[serde(skip)] + auto_import_data: Option, } impl CompletionEntry { + fn normalize(&mut self, specifier_map: &TscSpecifierMap) { + let Some(data) = &self.data else { + return; + }; + let Ok(raw) = + serde_json::from_value::(data.clone()) + else { + return; + }; + if let Ok(normalized) = specifier_map.normalize(&raw.file_name) { + self.auto_import_data = + Some(CompletionNormalizedAutoImportData { raw, normalized }); + } + } + fn get_commit_characters( &self, info: &CompletionInfo, @@ -3833,25 +3948,44 @@ impl CompletionEntry { if let Some(source) = &self.source { let mut display_source = source.clone(); - if let Some(data) = &self.data { - if let Ok(import_data) = - serde_json::from_value::(data.clone()) + if let Some(import_data) = &self.auto_import_data { + let import_mapper = + language_server.get_ts_response_import_mapper(specifier); + if let Some(mut new_specifier) = import_mapper + .check_specifier(&import_data.normalized, specifier) + .or_else(|| relative_specifier(specifier, &import_data.normalized)) { - if let Ok(import_specifier) = resolve_url(&import_data.file_name) { - if let Some(new_module_specifier) = language_server - .get_ts_response_import_mapper(specifier) - .check_specifier(&import_specifier, specifier) - .or_else(|| relative_specifier(specifier, &import_specifier)) - { - display_source.clone_from(&new_module_specifier); - if new_module_specifier != import_data.module_specifier { - specifier_rewrite = - Some((import_data.module_specifier, new_module_specifier)); - } - } else if source.starts_with(jsr_url().as_str()) { - return None; - } + if new_specifier.contains("/node_modules/") { + return None; } + let mut new_deno_types_specifier = None; + if let Some(code_specifier) = language_server + .resolver + .deno_types_to_code_resolution( + &import_data.normalized, + Some(specifier), + ) + .and_then(|s| { + import_mapper + .check_specifier(&s, specifier) + .or_else(|| relative_specifier(specifier, &s)) + }) + { + new_deno_types_specifier = + Some(std::mem::replace(&mut new_specifier, code_specifier)); + } + display_source.clone_from(&new_specifier); + if new_specifier != import_data.raw.module_specifier + || new_deno_types_specifier.is_some() + { + specifier_rewrite = Some(CompletionSpecifierRewrite { + old_specifier: import_data.raw.module_specifier.clone(), + new_specifier, + new_deno_types_specifier, + }); + } + } else if source.starts_with(jsr_url().as_str()) { + return None; } } // We want relative or bare (import-mapped or otherwise) specifiers to @@ -3939,7 +4073,7 @@ pub struct OutliningSpan { kind: OutliningSpanKind, } -const FOLD_END_PAIR_CHARACTERS: &[u8] = &[b'}', b']', b')', b'`']; +const FOLD_END_PAIR_CHARACTERS: &[u8] = b"}])`"; impl OutliningSpan { pub fn to_folding_range( @@ -4154,6 +4288,11 @@ impl TscSpecifierMap { return specifier.to_string(); } let mut specifier = original.to_string(); + if !specifier.contains("/node_modules/@types/node/") { + // The ts server doesn't give completions from files in + // `node_modules/.deno/`. We work around it like this. + specifier = specifier.replace("/node_modules/", "/$node_modules/"); + } let media_type = MediaType::from_specifier(original); // If the URL-inferred media type doesn't correspond to tsc's path-inferred // media type, force it to be the same by appending an extension. @@ -4271,7 +4410,7 @@ fn op_is_cancelled(state: &mut OpState) -> bool { fn op_is_node_file(state: &mut OpState, #[string] path: String) -> bool { let state = state.borrow::(); let mark = state.performance.mark("tsc.op.op_is_node_file"); - let r = match ModuleSpecifier::parse(&path) { + let r = match state.specifier_map.normalize(path) { Ok(specifier) => state.state_snapshot.resolver.in_node_modules(&specifier), Err(_) => false, }; @@ -4308,15 +4447,14 @@ fn op_load<'s>( data: doc.text(), script_kind: crate::tsc::as_ts_script_kind(doc.media_type()), version: state.script_version(&specifier), - is_cjs: matches!( - doc.media_type(), - MediaType::Cjs | MediaType::Cts | MediaType::Dcts - ), + is_cjs: doc + .document() + .map(|d| d.resolution_mode()) + .unwrap_or(ResolutionMode::Import) + == ResolutionMode::Require, }) }; - let serialized = serde_v8::to_v8(scope, maybe_load_response)?; - state.performance.measure(mark); Ok(serialized) } @@ -4341,17 +4479,9 @@ fn op_release( fn op_resolve( state: &mut OpState, #[string] base: String, - is_base_cjs: bool, - #[serde] specifiers: Vec, + #[serde] specifiers: Vec<(bool, String)>, ) -> Result>, AnyError> { - op_resolve_inner( - state, - ResolveArgs { - base, - is_base_cjs, - specifiers, - }, - ) + op_resolve_inner(state, ResolveArgs { base, specifiers }) } struct TscRequestArray { @@ -4540,7 +4670,10 @@ fn op_script_names(state: &mut OpState) -> ScriptNames { for doc in &docs { let specifier = doc.specifier(); let is_open = doc.is_open(); - if is_open || specifier.scheme() == "file" { + if is_open + || (specifier.scheme() == "file" + && !state.state_snapshot.resolver.in_node_modules(specifier)) + { let script_names = doc .scope() .and_then(|s| result.by_scope.get_mut(s)) @@ -4551,6 +4684,7 @@ fn op_script_names(state: &mut OpState) -> ScriptNames { let (types, _) = documents.resolve_dependency( types, specifier, + doc.resolution_mode(), doc.file_referrer(), )?; let types_doc = documents.get_or_load(&types, doc.file_referrer())?; @@ -4654,6 +4788,7 @@ fn run_tsc_thread( specifier_map, request_rx, )], + create_params: create_isolate_create_params(), startup_snapshot: Some(tsc::compiler_snapshot()), inspector: has_inspector_server, ..Default::default() @@ -4892,6 +5027,10 @@ pub struct UserPreferences { pub allow_rename_of_import_path: Option, #[serde(skip_serializing_if = "Option::is_none")] pub auto_import_file_exclude_patterns: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub interactive_inlay_hints: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub prefer_type_only_auto_imports: Option, } impl UserPreferences { @@ -4909,6 +5048,7 @@ impl UserPreferences { include_completions_with_snippet_text: Some( config.snippet_support_capable(), ), + interactive_inlay_hints: Some(true), provide_refactor_not_applicable_reason: Some(true), quote_preference: Some(fmt_config.into()), use_label_details_in_completion_entries: Some(true), @@ -5013,6 +5153,9 @@ impl UserPreferences { } else { Some(language_settings.preferences.quote_style) }, + prefer_type_only_auto_imports: Some( + language_settings.preferences.prefer_type_only_auto_imports, + ), ..base_preferences } } @@ -5450,7 +5593,7 @@ mod tests { let (_tx, rx) = mpsc::unbounded_channel(); let state = State::new(state_snapshot, Default::default(), Default::default(), rx); - let mut op_state = OpState::new(None); + let mut op_state = OpState::new(None, None); op_state.put(state); op_state } @@ -5958,6 +6101,7 @@ mod tests { Some(temp_dir.url()), ) .await + .unwrap() .unwrap(); assert_eq!(info.entries.len(), 22); let details = ts_server @@ -6117,6 +6261,7 @@ mod tests { Some(temp_dir.url()), ) .await + .unwrap() .unwrap(); let entry = info .entries @@ -6154,7 +6299,7 @@ mod tests { let change = changes.text_changes.first().unwrap(); assert_eq!( change.new_text, - "import type { someLongVariable } from './b.ts'\n" + "import { someLongVariable } from './b.ts'\n" ); } @@ -6273,8 +6418,7 @@ mod tests { &mut state, ResolveArgs { base: temp_dir.url().join("a.ts").unwrap().to_string(), - is_base_cjs: false, - specifiers: vec!["./b.ts".to_string()], + specifiers: vec![(false, "./b.ts".to_string())], }, ) .unwrap(); diff --git a/cli/main.rs b/cli/main.rs index c0eccab5df9c77..d47f1e363c4ba4 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -15,7 +15,6 @@ mod js; mod jsr; mod lsp; mod module_loader; -mod napi; mod node; mod npm; mod ops; @@ -37,6 +36,8 @@ use crate::util::v8::get_v8_flags_from_env; use crate::util::v8::init_v8_flags; use args::TaskFlags; +use deno_resolver::npm::ByonmResolvePkgFolderFromDenoReqError; +use deno_resolver::npm::ResolvePkgFolderFromDenoReqError; use deno_runtime::WorkerExecutionMode; pub use deno_runtime::UNSTABLE_GRANULAR_FLAGS; @@ -46,8 +47,7 @@ use deno_core::error::JsError; use deno_core::futures::FutureExt; use deno_core::unsync::JoinHandle; use deno_npm::resolution::SnapshotFromLockfileError; -use deno_runtime::fmt_errors::format_js_error_with_suggestions; -use deno_runtime::fmt_errors::FixSuggestion; +use deno_runtime::fmt_errors::format_js_error; use deno_runtime::tokio_util::create_and_run_current_thread_with_maybe_metrics; use deno_terminal::colors; use factory::CliFactory; @@ -55,10 +55,15 @@ use standalone::MODULE_NOT_FOUND; use standalone::UNSUPPORTED_SCHEME; use std::env; use std::future::Future; +use std::io::IsTerminal; use std::ops::Deref; use std::path::PathBuf; use std::sync::Arc; +#[cfg(feature = "dhat-heap")] +#[global_allocator] +static ALLOC: dhat::Alloc = dhat::Alloc; + /// Ensures that all subcommands return an i32 exit code and an [`AnyError`] error type. trait SubcommandOutput { fn output(self) -> Result; @@ -130,7 +135,7 @@ async fn run_subcommand(flags: Arc) -> Result { tools::compile::compile(flags, compile_flags).await }), DenoSubcommand::Coverage(coverage_flags) => spawn_subcommand(async { - tools::coverage::cover_files(flags, coverage_flags).await + tools::coverage::cover_files(flags, coverage_flags) }), DenoSubcommand::Fmt(fmt_flags) => { spawn_subcommand( @@ -139,9 +144,7 @@ async fn run_subcommand(flags: Arc) -> Result { } DenoSubcommand::Init(init_flags) => { spawn_subcommand(async { - // make compiler happy since init_project is sync - tokio::task::yield_now().await; - tools::init::init_project(init_flags) + tools::init::init_project(init_flags).await }) } DenoSubcommand::Info(info_flags) => { @@ -159,7 +162,19 @@ async fn run_subcommand(flags: Arc) -> Result { DenoSubcommand::Uninstall(uninstall_flags) => spawn_subcommand(async { tools::installer::uninstall(flags, uninstall_flags).await }), - DenoSubcommand::Lsp => spawn_subcommand(async { lsp::start().await }), + DenoSubcommand::Lsp => spawn_subcommand(async { + if std::io::stderr().is_terminal() { + log::warn!( + "{} command is intended to be run by text editors and IDEs and shouldn't be run manually. + + Visit https://docs.deno.com/runtime/getting_started/setup_your_environment/ for instruction + how to setup your favorite text editor. + + Press Ctrl+C to exit. + ", colors::cyan("deno lsp")); + } + lsp::start().await + }), DenoSubcommand::Lint(lint_flags) => spawn_subcommand(async { if lint_flags.rules { tools::lint::print_rules_list( @@ -171,6 +186,11 @@ async fn run_subcommand(flags: Arc) -> Result { tools::lint::lint(flags, lint_flags).await } }), + DenoSubcommand::Outdated(update_flags) => { + spawn_subcommand(async move { + tools::registry::outdated(flags, update_flags).await + }) + } DenoSubcommand::Repl(repl_flags) => { spawn_subcommand(async move { tools::repl::run(flags, repl_flags).await }) } @@ -182,6 +202,21 @@ async fn run_subcommand(flags: Arc) -> Result { match result { Ok(v) => Ok(v), Err(script_err) => { + if let Some(ResolvePkgFolderFromDenoReqError::Byonm(ByonmResolvePkgFolderFromDenoReqError::UnmatchedReq(_))) = script_err.downcast_ref::() { + if flags.node_modules_dir.is_none() { + let mut flags = flags.deref().clone(); + let watch = match &flags.subcommand { + DenoSubcommand::Run(run_flags) => run_flags.watch.clone(), + _ => unreachable!(), + }; + flags.node_modules_dir = Some(deno_config::deno_json::NodeModulesDirMode::None); + // use the current lockfile, but don't write it out + if flags.frozen_lockfile.is_none() { + flags.internal.lockfile_skip_write = true; + } + return tools::run::run_script(WorkerExecutionMode::Run, Arc::new(flags), watch).await; + } + } let script_err_msg = script_err.to_string(); if script_err_msg.starts_with(MODULE_NOT_FOUND) || script_err_msg.starts_with(UNSUPPORTED_SCHEME) { if run_flags.bare { @@ -206,6 +241,9 @@ async fn run_subcommand(flags: Arc) -> Result { cwd: None, task: Some(run_flags.script.clone()), is_run: true, + recursive: false, + filter: None, + eval: false, }; new_flags.subcommand = DenoSubcommand::Task(task_flags.clone()); let result = tools::task::execute_script(Arc::new(new_flags), task_flags.clone()).await; @@ -318,84 +356,17 @@ fn setup_panic_hook() { eprintln!("Args: {:?}", env::args().collect::>()); eprintln!(); orig_hook(panic_info); - std::process::exit(1); + deno_runtime::exit(1); })); } -#[allow(clippy::print_stderr)] fn exit_with_message(message: &str, code: i32) -> ! { - eprintln!( + log::error!( "{}: {}", colors::red_bold("error"), message.trim_start_matches("error: ") ); - std::process::exit(code); -} - -fn get_suggestions_for_terminal_errors(e: &JsError) -> Vec { - if let Some(msg) = &e.message { - if msg.contains("module is not defined") - || msg.contains("exports is not defined") - { - return vec![ - FixSuggestion::info( - "Deno does not support CommonJS modules without `.cjs` extension.", - ), - FixSuggestion::hint( - "Rewrite this module to ESM or change the file extension to `.cjs`.", - ), - ]; - } else if msg.contains("openKv is not a function") { - return vec![ - FixSuggestion::info("Deno.openKv() is an unstable API."), - FixSuggestion::hint( - "Run again with `--unstable-kv` flag to enable this API.", - ), - ]; - } else if msg.contains("cron is not a function") { - return vec![ - FixSuggestion::info("Deno.cron() is an unstable API."), - FixSuggestion::hint( - "Run again with `--unstable-cron` flag to enable this API.", - ), - ]; - } else if msg.contains("WebSocketStream is not defined") { - return vec![ - FixSuggestion::info("new WebSocketStream() is an unstable API."), - FixSuggestion::hint( - "Run again with `--unstable-net` flag to enable this API.", - ), - ]; - } else if msg.contains("Temporal is not defined") { - return vec![ - FixSuggestion::info("Temporal is an unstable API."), - FixSuggestion::hint( - "Run again with `--unstable-temporal` flag to enable this API.", - ), - ]; - } else if msg.contains("BroadcastChannel is not defined") { - return vec![ - FixSuggestion::info("BroadcastChannel is an unstable API."), - FixSuggestion::hint( - "Run again with `--unstable-broadcast-channel` flag to enable this API.", - ), - ]; - } else if msg.contains("window is not defined") { - return vec![ - FixSuggestion::info("window global is not available in Deno 2."), - FixSuggestion::hint("Replace `window` with `globalThis`."), - ]; - } else if msg.contains("UnsafeWindowSurface is not a constructor") { - return vec![ - FixSuggestion::info("Deno.UnsafeWindowSurface is an unstable API."), - FixSuggestion::hint( - "Run again with `--unstable-webgpu` flag to enable this API.", - ), - ]; - } - } - - vec![] + deno_runtime::exit(code); } fn exit_for_error(error: AnyError) -> ! { @@ -403,8 +374,7 @@ fn exit_for_error(error: AnyError) -> ! { let mut error_code = 1; if let Some(e) = error.downcast_ref::() { - let suggestions = get_suggestions_for_terminal_errors(e); - error_string = format_js_error_with_suggestions(e, suggestions); + error_string = format_js_error(e); } else if let Some(SnapshotFromLockfileError::IntegrityCheckFailed(e)) = error.downcast_ref::() { @@ -415,16 +385,18 @@ fn exit_for_error(error: AnyError) -> ! { exit_with_message(&error_string, error_code); } -#[allow(clippy::print_stderr)] pub(crate) fn unstable_exit_cb(feature: &str, api_name: &str) { - eprintln!( + log::error!( "Unstable API '{api_name}'. The `--unstable-{}` flag must be provided.", feature ); - std::process::exit(70); + deno_runtime::exit(70); } pub fn main() { + #[cfg(feature = "dhat-heap")] + let profiler = dhat::Profiler::new_heap(); + setup_panic_hook(); util::unix::raise_fd_limit(); @@ -445,8 +417,13 @@ pub fn main() { run_subcommand(Arc::new(flags)).await }; - match create_and_run_current_thread_with_maybe_metrics(future) { - Ok(exit_code) => std::process::exit(exit_code), + let result = create_and_run_current_thread_with_maybe_metrics(future); + + #[cfg(feature = "dhat-heap")] + drop(profiler); + + match result { + Ok(exit_code) => deno_runtime::exit(exit_code), Err(err) => exit_for_error(err), } } @@ -460,12 +437,21 @@ fn resolve_flags_and_init( if err.kind() == clap::error::ErrorKind::DisplayVersion => { // Ignore results to avoid BrokenPipe errors. + util::logger::init(None); let _ = err.print(); - std::process::exit(0); + deno_runtime::exit(0); + } + Err(err) => { + util::logger::init(None); + exit_for_error(AnyError::from(err)) } - Err(err) => exit_for_error(AnyError::from(err)), }; + if let Some(otel_config) = flags.otel_config() { + deno_telemetry::init(otel_config)?; + } + util::logger::init(flags.log_level); + // TODO(bartlomieju): remove in Deno v2.5 and hard error then. if flags.unstable_config.legacy_flag_enabled { log::warn!( @@ -494,7 +480,6 @@ fn resolve_flags_and_init( deno_core::JsRuntime::init_platform( None, /* import assertions enabled */ false, ); - util::logger::init(flags.log_level); Ok(flags) } diff --git a/cli/mainrt.rs b/cli/mainrt.rs index 02d58fcee1500f..7ad3b3744b075b 100644 --- a/cli/mainrt.rs +++ b/cli/mainrt.rs @@ -40,23 +40,21 @@ use std::env::current_exe; use crate::args::Flags; -#[allow(clippy::print_stderr)] pub(crate) fn unstable_exit_cb(feature: &str, api_name: &str) { - eprintln!( + log::error!( "Unstable API '{api_name}'. The `--unstable-{}` flag must be provided.", feature ); - std::process::exit(70); + deno_runtime::exit(70); } -#[allow(clippy::print_stderr)] fn exit_with_message(message: &str, code: i32) -> ! { - eprintln!( + log::error!( "{}: {}", colors::red_bold("error"), message.trim_start_matches("error: ") ); - std::process::exit(code); + deno_runtime::exit(code); } fn unwrap_or_exit(result: Result) -> T { @@ -88,15 +86,20 @@ fn main() { let standalone = standalone::extract_standalone(Cow::Owned(args)); let future = async move { match standalone { - Ok(Some(future)) => { - let (metadata, eszip) = future.await?; - util::logger::init(metadata.log_level); - load_env_vars(&metadata.env_vars_from_env_file); - let exit_code = standalone::run(eszip, metadata).await?; - std::process::exit(exit_code); + Ok(Some(data)) => { + if let Some(otel_config) = data.metadata.otel_config.clone() { + deno_telemetry::init(otel_config)?; + } + util::logger::init(data.metadata.log_level); + load_env_vars(&data.metadata.env_vars_from_env_file); + let exit_code = standalone::run(data).await?; + deno_runtime::exit(exit_code); } Ok(None) => Ok(()), - Err(err) => Err(err), + Err(err) => { + util::logger::init(None); + Err(err) + } } }; diff --git a/cli/module_loader.rs b/cli/module_loader.rs index a81c5a0aa8a7c6..447c85a9ac573f 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -2,6 +2,7 @@ use std::borrow::Cow; use std::cell::RefCell; +use std::path::Path; use std::path::PathBuf; use std::pin::Pin; use std::rc::Rc; @@ -23,18 +24,23 @@ use crate::graph_container::ModuleGraphUpdatePermit; use crate::graph_util::CreateGraphOptions; use crate::graph_util::ModuleGraphBuilder; use crate::node; -use crate::resolver::CliGraphResolver; -use crate::resolver::CliNodeResolver; +use crate::node::CliNodeCodeTranslator; +use crate::npm::CliNpmResolver; +use crate::resolver::CjsTracker; +use crate::resolver::CliNpmReqResolver; +use crate::resolver::CliResolver; use crate::resolver::ModuleCodeStringSource; +use crate::resolver::NotSupportedKindInNpmError; use crate::resolver::NpmModuleLoader; use crate::tools::check; use crate::tools::check::TypeChecker; use crate::util::progress_bar::ProgressBar; use crate::util::text_encoding::code_without_source_map; use crate::util::text_encoding::source_map_from_code; -use crate::worker::ModuleLoaderAndSourceMapGetter; +use crate::worker::CreateModuleLoaderResult; use crate::worker::ModuleLoaderFactory; use deno_ast::MediaType; +use deno_ast::ModuleKind; use deno_core::anyhow::anyhow; use deno_core::anyhow::bail; use deno_core::anyhow::Context; @@ -51,21 +57,25 @@ use deno_core::ModuleSourceCode; use deno_core::ModuleSpecifier; use deno_core::ModuleType; use deno_core::RequestedModuleType; -use deno_core::ResolutionKind; use deno_core::SourceCodeCacheInfo; -use deno_graph::source::ResolutionMode; -use deno_graph::source::Resolver; use deno_graph::GraphKind; use deno_graph::JsModule; use deno_graph::JsonModule; use deno_graph::Module; use deno_graph::ModuleGraph; use deno_graph::Resolution; +use deno_graph::WasmModule; use deno_runtime::code_cache; +use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_node::create_host_defined_options; +use deno_runtime::deno_node::NodeRequireLoader; +use deno_runtime::deno_node::NodeResolver; use deno_runtime::deno_permissions::PermissionsContainer; use deno_semver::npm::NpmPackageReqReference; -use node_resolver::NodeResolutionMode; +use node_resolver::errors::ClosestPkgJsonError; +use node_resolver::InNpmPackageChecker; +use node_resolver::NodeResolutionKind; +use node_resolver::ResolutionMode; pub struct ModuleLoadPreparer { options: Arc, @@ -104,7 +114,7 @@ impl ModuleLoadPreparer { roots: &[ModuleSpecifier], is_dynamic: bool, lib: TsTypeLib, - permissions: crate::file_fetcher::FetchPermissionsOption, + permissions: PermissionsContainer, ext_overwrite: Option<&String>, ) -> Result<(), AnyError> { log::debug!("Preparing module load."); @@ -198,14 +208,20 @@ struct SharedCliModuleLoaderState { initial_cwd: PathBuf, is_inspecting: bool, is_repl: bool, + cjs_tracker: Arc, code_cache: Option>, emitter: Arc, + fs: Arc, + in_npm_pkg_checker: Arc, main_module_graph_container: Arc, module_load_preparer: Arc, - node_resolver: Arc, + node_code_translator: Arc, + node_resolver: Arc, + npm_req_resolver: Arc, + npm_resolver: Arc, npm_module_loader: NpmModuleLoader, parsed_source_cache: Arc, - resolver: Arc, + resolver: Arc, } pub struct CliModuleLoaderFactory { @@ -216,14 +232,20 @@ impl CliModuleLoaderFactory { #[allow(clippy::too_many_arguments)] pub fn new( options: &CliOptions, + cjs_tracker: Arc, code_cache: Option>, emitter: Arc, + fs: Arc, + in_npm_pkg_checker: Arc, main_module_graph_container: Arc, module_load_preparer: Arc, - node_resolver: Arc, + node_code_translator: Arc, + node_resolver: Arc, + npm_req_resolver: Arc, + npm_resolver: Arc, npm_module_loader: NpmModuleLoader, parsed_source_cache: Arc, - resolver: Arc, + resolver: Arc, ) -> Self { Self { shared: Arc::new(SharedCliModuleLoaderState { @@ -236,11 +258,17 @@ impl CliModuleLoaderFactory { options.sub_command(), DenoSubcommand::Repl(_) | DenoSubcommand::Jupyter(_) ), + cjs_tracker, code_cache, emitter, + fs, + in_npm_pkg_checker, main_module_graph_container, module_load_preparer, + node_code_translator, node_resolver, + npm_req_resolver, + npm_resolver, npm_module_loader, parsed_source_cache, resolver, @@ -252,20 +280,33 @@ impl CliModuleLoaderFactory { &self, graph_container: TGraphContainer, lib: TsTypeLib, - root_permissions: PermissionsContainer, - dynamic_permissions: PermissionsContainer, - ) -> ModuleLoaderAndSourceMapGetter { - let loader = Rc::new(CliModuleLoader(Rc::new(CliModuleLoaderInner { - lib, - root_permissions, - dynamic_permissions, - graph_container, + is_worker: bool, + parent_permissions: PermissionsContainer, + permissions: PermissionsContainer, + ) -> CreateModuleLoaderResult { + let module_loader = + Rc::new(CliModuleLoader(Rc::new(CliModuleLoaderInner { + lib, + is_worker, + parent_permissions, + permissions, + graph_container: graph_container.clone(), + node_code_translator: self.shared.node_code_translator.clone(), + emitter: self.shared.emitter.clone(), + parsed_source_cache: self.shared.parsed_source_cache.clone(), + shared: self.shared.clone(), + }))); + let node_require_loader = Rc::new(CliNodeRequireLoader { + cjs_tracker: self.shared.cjs_tracker.clone(), emitter: self.shared.emitter.clone(), - parsed_source_cache: self.shared.parsed_source_cache.clone(), - shared: self.shared.clone(), - }))); - ModuleLoaderAndSourceMapGetter { - module_loader: loader, + fs: self.shared.fs.clone(), + graph_container, + in_npm_pkg_checker: self.shared.in_npm_pkg_checker.clone(), + npm_resolver: self.shared.npm_resolver.clone(), + }); + CreateModuleLoaderResult { + module_loader, + node_require_loader, } } } @@ -274,44 +315,45 @@ impl ModuleLoaderFactory for CliModuleLoaderFactory { fn create_for_main( &self, root_permissions: PermissionsContainer, - dynamic_permissions: PermissionsContainer, - ) -> ModuleLoaderAndSourceMapGetter { + ) -> CreateModuleLoaderResult { self.create_with_lib( (*self.shared.main_module_graph_container).clone(), self.shared.lib_window, + /* is worker */ false, + root_permissions.clone(), root_permissions, - dynamic_permissions, ) } fn create_for_worker( &self, - root_permissions: PermissionsContainer, - dynamic_permissions: PermissionsContainer, - ) -> ModuleLoaderAndSourceMapGetter { + parent_permissions: PermissionsContainer, + permissions: PermissionsContainer, + ) -> CreateModuleLoaderResult { self.create_with_lib( // create a fresh module graph for the worker WorkerModuleGraphContainer::new(Arc::new(ModuleGraph::new( self.shared.graph_kind, ))), self.shared.lib_worker, - root_permissions, - dynamic_permissions, + /* is worker */ true, + parent_permissions, + permissions, ) } } struct CliModuleLoaderInner { lib: TsTypeLib, + is_worker: bool, /// The initial set of permissions used to resolve the static imports in the /// worker. These are "allow all" for main worker, and parent thread /// permissions for Web Worker. - root_permissions: PermissionsContainer, - /// Permissions used to resolve dynamic imports, these get passed as - /// "root permissions" for Web Worker. - dynamic_permissions: PermissionsContainer, + parent_permissions: PermissionsContainer, + permissions: PermissionsContainer, shared: Arc, emitter: Arc, + node_code_translator: Arc, parsed_source_cache: Arc, graph_container: TGraphContainer, } @@ -325,17 +367,10 @@ impl maybe_referrer: Option<&ModuleSpecifier>, requested_module_type: RequestedModuleType, ) -> Result { - let code_source = if let Some(result) = self - .shared - .npm_module_loader - .load_if_in_npm_package(specifier, maybe_referrer) - .await + let code_source = self.load_code_source(specifier, maybe_referrer).await?; + let code = if self.shared.is_inspecting + || code_source.media_type == MediaType::Wasm { - result? - } else { - self.load_prepared_module(specifier, maybe_referrer).await? - }; - let code = if self.shared.is_inspecting { // we need the code with the source map in order for // it to work with --inspect or --inspect-brk code_source.code @@ -345,6 +380,7 @@ impl }; let module_type = match code_source.media_type { MediaType::Json => ModuleType::Json, + MediaType::Wasm => ModuleType::Wasm, _ => ModuleType::JavaScript, }; @@ -388,6 +424,29 @@ impl )) } + async fn load_code_source( + &self, + specifier: &ModuleSpecifier, + maybe_referrer: Option<&ModuleSpecifier>, + ) -> Result { + if let Some(code_source) = self.load_prepared_module(specifier).await? { + return Ok(code_source); + } + if self.shared.in_npm_pkg_checker.in_npm_package(specifier) { + return self + .shared + .npm_module_loader + .load(specifier, maybe_referrer) + .await; + } + + let mut msg = format!("Loading unprepared module: {specifier}"); + if let Some(referrer) = maybe_referrer { + msg = format!("{}, imported from: {}", msg, referrer.as_str()); + } + Err(anyhow!(msg)) + } + fn resolve_referrer( &self, referrer: &str, @@ -395,7 +454,7 @@ impl let referrer = if referrer.is_empty() && self.shared.is_repl { // FIXME(bartlomieju): this is a hacky way to provide compatibility with REPL // and `Deno.core.evalContext` API. Ideally we should always have a referrer filled - "./$deno$repl.ts" + "./$deno$repl.mts" } else { referrer }; @@ -418,16 +477,6 @@ impl raw_specifier: &str, referrer: &ModuleSpecifier, ) -> Result { - if self.shared.node_resolver.in_npm_package(referrer) { - return Ok( - self - .shared - .node_resolver - .resolve(raw_specifier, referrer, NodeResolutionMode::Execution)? - .into_url(), - ); - } - let graph = self.graph_container.graph(); let resolution = match graph.get(referrer) { Some(Module::Js(module)) => module @@ -448,12 +497,11 @@ impl } Resolution::None => Cow::Owned(self.shared.resolver.resolve( raw_specifier, - &deno_graph::Range { - specifier: referrer.clone(), - start: deno_graph::Position::zeroed(), - end: deno_graph::Position::zeroed(), - }, - ResolutionMode::Execution, + referrer, + deno_graph::Position::zeroed(), + // if we're here, that means it's resolving a dynamic import + ResolutionMode::Import, + NodeResolutionKind::Execution, )?), }; @@ -462,13 +510,14 @@ impl { return self .shared - .node_resolver + .npm_req_resolver .resolve_req_reference( &reference, referrer, - NodeResolutionMode::Execution, + ResolutionMode::Import, + NodeResolutionKind::Execution, ) - .map(|res| res.into_url()); + .map_err(AnyError::from); } } @@ -476,7 +525,6 @@ impl Some(Module::Npm(module)) => { let package_folder = self .shared - .node_resolver .npm_resolver .as_managed() .unwrap() // byonm won't create a Module::Npm @@ -484,22 +532,26 @@ impl self .shared .node_resolver - .resolve_package_sub_path_from_deno_module( + .resolve_package_subpath_from_deno_module( &package_folder, module.nv_reference.sub_path(), Some(referrer), - NodeResolutionMode::Execution, + ResolutionMode::Import, + NodeResolutionKind::Execution, ) .with_context(|| { format!("Could not resolve '{}'.", module.nv_reference) })? - .into_url() } Some(Module::Node(module)) => module.specifier.clone(), Some(Module::Js(module)) => module.specifier.clone(), Some(Module::Json(module)) => module.specifier.clone(), + Some(Module::Wasm(module)) => module.specifier.clone(), Some(Module::External(module)) => { - node::resolve_specifier_into_node_modules(&module.specifier) + node::resolve_specifier_into_node_modules( + &module.specifier, + self.shared.fs.as_ref(), + ) } None => specifier.into_owned(), }; @@ -509,71 +561,82 @@ impl async fn load_prepared_module( &self, specifier: &ModuleSpecifier, - maybe_referrer: Option<&ModuleSpecifier>, - ) -> Result { + ) -> Result, AnyError> { // Note: keep this in sync with the sync version below let graph = self.graph_container.graph(); - match self.load_prepared_module_or_defer_emit( - &graph, - specifier, - maybe_referrer, - ) { - Ok(CodeOrDeferredEmit::Code(code_source)) => Ok(code_source), - Ok(CodeOrDeferredEmit::DeferredEmit { + match self.load_prepared_module_or_defer_emit(&graph, specifier)? { + Some(CodeOrDeferredEmit::Code(code_source)) => Ok(Some(code_source)), + Some(CodeOrDeferredEmit::DeferredEmit { specifier, media_type, source, }) => { let transpile_result = self .emitter - .emit_parsed_source(specifier, media_type, source) + .emit_parsed_source(specifier, media_type, ModuleKind::Esm, source) .await?; // at this point, we no longer need the parsed source in memory, so free it self.parsed_source_cache.free(specifier); - Ok(ModuleCodeStringSource { - code: ModuleSourceCode::Bytes(transpile_result), + Ok(Some(ModuleCodeStringSource { + // note: it's faster to provide a string if we know it's a string + code: ModuleSourceCode::String(transpile_result.into()), found_url: specifier.clone(), media_type, - }) + })) } - Err(err) => Err(err), + Some(CodeOrDeferredEmit::Cjs { + specifier, + media_type, + source, + }) => self + .load_maybe_cjs(specifier, media_type, source) + .await + .map(Some), + None => Ok(None), } } - fn load_prepared_module_sync( + fn load_prepared_module_for_source_map_sync( &self, specifier: &ModuleSpecifier, - maybe_referrer: Option<&ModuleSpecifier>, - ) -> Result { + ) -> Result, AnyError> { // Note: keep this in sync with the async version above let graph = self.graph_container.graph(); - match self.load_prepared_module_or_defer_emit( - &graph, - specifier, - maybe_referrer, - ) { - Ok(CodeOrDeferredEmit::Code(code_source)) => Ok(code_source), - Ok(CodeOrDeferredEmit::DeferredEmit { + match self.load_prepared_module_or_defer_emit(&graph, specifier)? { + Some(CodeOrDeferredEmit::Code(code_source)) => Ok(Some(code_source)), + Some(CodeOrDeferredEmit::DeferredEmit { specifier, media_type, source, }) => { - let transpile_result = self - .emitter - .emit_parsed_source_sync(specifier, media_type, source)?; + let transpile_result = self.emitter.emit_parsed_source_sync( + specifier, + media_type, + ModuleKind::Esm, + source, + )?; // at this point, we no longer need the parsed source in memory, so free it self.parsed_source_cache.free(specifier); - Ok(ModuleCodeStringSource { - code: ModuleSourceCode::Bytes(transpile_result), + Ok(Some(ModuleCodeStringSource { + // note: it's faster to provide a string if we know it's a string + code: ModuleSourceCode::String(transpile_result.into()), found_url: specifier.clone(), media_type, - }) + })) + } + Some(CodeOrDeferredEmit::Cjs { .. }) => { + self.parsed_source_cache.free(specifier); + + // todo(dsherret): to make this work, we should probably just + // rely on the CJS export cache. At the moment this is hard because + // cjs export analysis is only async + Ok(None) } - Err(err) => Err(err), + None => Ok(None), } } @@ -581,8 +644,7 @@ impl &self, graph: &'graph ModuleGraph, specifier: &ModuleSpecifier, - maybe_referrer: Option<&ModuleSpecifier>, - ) -> Result, AnyError> { + ) -> Result>, AnyError> { if specifier.scheme() == "node" { // Node built-in modules should be handled internally. unreachable!("Deno bug. {} was misconfigured internally.", specifier); @@ -594,38 +656,55 @@ impl media_type, specifier, .. - })) => Ok(CodeOrDeferredEmit::Code(ModuleCodeStringSource { + })) => Ok(Some(CodeOrDeferredEmit::Code(ModuleCodeStringSource { code: ModuleSourceCode::String(source.clone().into()), found_url: specifier.clone(), media_type: *media_type, - })), + }))), Some(deno_graph::Module::Js(JsModule { source, media_type, specifier, + is_script, .. })) => { + if self.shared.cjs_tracker.is_cjs_with_known_is_script( + specifier, + *media_type, + *is_script, + )? { + return Ok(Some(CodeOrDeferredEmit::Cjs { + specifier, + media_type: *media_type, + source, + })); + } let code: ModuleCodeString = match media_type { MediaType::JavaScript | MediaType::Unknown - | MediaType::Cjs | MediaType::Mjs | MediaType::Json => source.clone().into(), MediaType::Dts | MediaType::Dcts | MediaType::Dmts => { Default::default() } + MediaType::Cjs | MediaType::Cts => { + return Ok(Some(CodeOrDeferredEmit::Cjs { + specifier, + media_type: *media_type, + source, + })); + } MediaType::TypeScript | MediaType::Mts - | MediaType::Cts | MediaType::Jsx | MediaType::Tsx => { - return Ok(CodeOrDeferredEmit::DeferredEmit { + return Ok(Some(CodeOrDeferredEmit::DeferredEmit { specifier, media_type: *media_type, source, - }); + })); } - MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => { + MediaType::Css | MediaType::Wasm | MediaType::SourceMap => { panic!("Unexpected media type {media_type} for {specifier}") } }; @@ -633,26 +712,69 @@ impl // at this point, we no longer need the parsed source in memory, so free it self.parsed_source_cache.free(specifier); - Ok(CodeOrDeferredEmit::Code(ModuleCodeStringSource { + Ok(Some(CodeOrDeferredEmit::Code(ModuleCodeStringSource { code: ModuleSourceCode::String(code), found_url: specifier.clone(), media_type: *media_type, - })) + }))) } + Some(deno_graph::Module::Wasm(WasmModule { + source, specifier, .. + })) => Ok(Some(CodeOrDeferredEmit::Code(ModuleCodeStringSource { + code: ModuleSourceCode::Bytes(source.clone().into()), + found_url: specifier.clone(), + media_type: MediaType::Wasm, + }))), Some( deno_graph::Module::External(_) | deno_graph::Module::Node(_) | deno_graph::Module::Npm(_), ) - | None => { - let mut msg = format!("Loading unprepared module: {specifier}"); - if let Some(referrer) = maybe_referrer { - msg = format!("{}, imported from: {}", msg, referrer.as_str()); - } - Err(anyhow!(msg)) - } + | None => Ok(None), } } + + async fn load_maybe_cjs( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + original_source: &Arc, + ) -> Result { + let js_source = if media_type.is_emittable() { + Cow::Owned( + self + .emitter + .emit_parsed_source( + specifier, + media_type, + ModuleKind::Cjs, + original_source, + ) + .await?, + ) + } else { + Cow::Borrowed(original_source.as_ref()) + }; + let text = self + .node_code_translator + .translate_cjs_to_esm(specifier, Some(js_source)) + .await?; + // at this point, we no longer need the parsed source in memory, so free it + self.parsed_source_cache.free(specifier); + Ok(ModuleCodeStringSource { + code: match text { + // perf: if the text is borrowed, that means it didn't make any changes + // to the original source, so we can just provide that instead of cloning + // the borrowed text + Cow::Borrowed(_) => { + ModuleSourceCode::String(original_source.clone().into()) + } + Cow::Owned(text) => ModuleSourceCode::String(text.into()), + }, + found_url: specifier.clone(), + media_type, + }) + } } enum CodeOrDeferredEmit<'a> { @@ -662,6 +784,11 @@ enum CodeOrDeferredEmit<'a> { media_type: MediaType, source: &'a Arc, }, + Cjs { + specifier: &'a ModuleSpecifier, + media_type: MediaType, + source: &'a Arc, + }, } // todo(dsherret): this double Rc boxing is not ideal @@ -676,7 +803,7 @@ impl ModuleLoader &self, specifier: &str, referrer: &str, - _kind: ResolutionKind, + _kind: deno_core::ResolutionKind, ) -> Result { fn ensure_not_jsr_non_jsr_remote_import( specifier: &ModuleSpecifier, @@ -703,7 +830,7 @@ impl ModuleLoader name: &str, ) -> Option> { let name = deno_core::ModuleSpecifier::parse(name).ok()?; - if self.0.shared.node_resolver.in_npm_package(&name) { + if self.0.shared.in_npm_pkg_checker.in_npm_package(&name) { Some(create_host_defined_options(scope)) } else { None @@ -740,7 +867,7 @@ impl ModuleLoader _maybe_referrer: Option, is_dynamic: bool, ) -> Pin>>> { - if self.0.shared.node_resolver.in_npm_package(specifier) { + if self.0.shared.in_npm_pkg_checker.in_npm_package(specifier) { return Box::pin(deno_core::futures::future::ready(Ok(()))); } @@ -769,11 +896,12 @@ impl ModuleLoader } } - let root_permissions = if is_dynamic { - inner.dynamic_permissions.clone() + let permissions = if is_dynamic { + inner.permissions.clone() } else { - inner.root_permissions.clone() + inner.parent_permissions.clone() }; + let is_dynamic = is_dynamic || inner.is_worker; // consider workers as dynamic for permissions let lib = inner.lib; let mut update_permit = graph_container.acquire_update_permit().await; let graph = update_permit.graph_mut(); @@ -783,7 +911,7 @@ impl ModuleLoader &[specifier], is_dynamic, lib, - root_permissions.into(), + permissions, None, ) .await?; @@ -822,7 +950,10 @@ impl ModuleLoader "wasm" | "file" | "http" | "https" | "data" | "blob" => (), _ => return None, } - let source = self.0.load_prepared_module_sync(&specifier, None).ok()?; + let source = self + .0 + .load_prepared_module_for_source_map_sync(&specifier) + .ok()??; source_map_from_code(source.code.as_bytes()) } @@ -901,3 +1032,74 @@ impl ModuleGraphUpdatePermit for WorkerModuleGraphUpdatePermit { drop(self.permit); // explicit drop for clarity } } + +#[derive(Debug)] +struct CliNodeRequireLoader { + cjs_tracker: Arc, + emitter: Arc, + fs: Arc, + graph_container: TGraphContainer, + in_npm_pkg_checker: Arc, + npm_resolver: Arc, +} + +impl NodeRequireLoader + for CliNodeRequireLoader +{ + fn ensure_read_permission<'a>( + &self, + permissions: &mut dyn deno_runtime::deno_node::NodePermissions, + path: &'a Path, + ) -> Result, AnyError> { + if let Ok(url) = deno_path_util::url_from_file_path(path) { + // allow reading if it's in the module graph + if self.graph_container.graph().get(&url).is_some() { + return Ok(std::borrow::Cow::Borrowed(path)); + } + } + self.npm_resolver.ensure_read_permission(permissions, path) + } + + fn load_text_file_lossy( + &self, + path: &Path, + ) -> Result, AnyError> { + // todo(dsherret): use the preloaded module from the graph if available? + let media_type = MediaType::from_path(path); + let text = self.fs.read_text_file_lossy_sync(path, None)?; + if media_type.is_emittable() { + let specifier = deno_path_util::url_from_file_path(path)?; + if self.in_npm_pkg_checker.in_npm_package(&specifier) { + return Err( + NotSupportedKindInNpmError { + media_type, + specifier, + } + .into(), + ); + } + self + .emitter + .emit_parsed_source_sync( + &specifier, + media_type, + // this is probably not super accurate due to require esm, but probably ok. + // If we find this causes a lot of churn in the emit cache then we should + // investigate how we can make this better + ModuleKind::Cjs, + &text.into(), + ) + .map(Cow::Owned) + } else { + Ok(text) + } + } + + fn is_maybe_cjs( + &self, + specifier: &ModuleSpecifier, + ) -> Result { + let media_type = MediaType::from_specifier(specifier); + self.cjs_tracker.is_maybe_cjs(specifier, media_type) + } +} diff --git a/cli/napi/README.md b/cli/napi/README.md deleted file mode 100644 index 7b359ac6ecc267..00000000000000 --- a/cli/napi/README.md +++ /dev/null @@ -1,114 +0,0 @@ -# napi - -This directory contains source for Deno's Node-API implementation. It depends on -`napi_sym` and `deno_napi`. - -Files are generally organized the same as in Node.js's implementation to ease in -ensuring compatibility. - -## Adding a new function - -Add the symbol name to -[`cli/napi_sym/symbol_exports.json`](../napi_sym/symbol_exports.json). - -```diff -{ - "symbols": [ - ... - "napi_get_undefined", -- "napi_get_null" -+ "napi_get_null", -+ "napi_get_boolean" - ] -} -``` - -Determine where to place the implementation. `napi_get_boolean` is related to JS -values so we will place it in `js_native_api.rs`. If something is not clear, -just create a new file module. - -See [`napi_sym`](../napi_sym/) for writing the implementation: - -```rust -#[napi_sym::napi_sym] -pub fn napi_get_boolean( - env: *mut Env, - value: bool, - result: *mut napi_value, -) -> Result { - // ... - Ok(()) -} -``` - -Update the generated symbol lists using the script: - -``` -deno run --allow-write tools/napi/generate_symbols_lists.js -``` - -Add a test in [`/tests/napi`](../../tests/napi/). You can also refer to Node.js -test suite for Node-API. - -```js -// tests/napi/boolean_test.js -import { assertEquals, loadTestLibrary } from "./common.js"; -const lib = loadTestLibrary(); -Deno.test("napi get boolean", function () { - assertEquals(lib.test_get_boolean(true), true); - assertEquals(lib.test_get_boolean(false), false); -}); -``` - -```rust -// tests/napi/src/boolean.rs - -use napi_sys::Status::napi_ok; -use napi_sys::ValueType::napi_boolean; -use napi_sys::*; - -extern "C" fn test_boolean( - env: napi_env, - info: napi_callback_info, -) -> napi_value { - let (args, argc, _) = crate::get_callback_info!(env, info, 1); - assert_eq!(argc, 1); - - let mut ty = -1; - assert!(unsafe { napi_typeof(env, args[0], &mut ty) } == napi_ok); - assert_eq!(ty, napi_boolean); - - // Use napi_get_boolean here... - - value -} - -pub fn init(env: napi_env, exports: napi_value) { - let properties = &[crate::new_property!(env, "test_boolean\0", test_boolean)]; - - unsafe { - napi_define_properties(env, exports, properties.len(), properties.as_ptr()) - }; -} -``` - -```diff -// tests/napi/src/lib.rs - -+ mod boolean; - -... - -#[no_mangle] -unsafe extern "C" fn napi_register_module_v1( - env: napi_env, - exports: napi_value, -) -> napi_value { - ... -+ boolean::init(env, exports); - - exports -} -``` - -Run the test using `cargo test -p tests/napi`. diff --git a/cli/napi/mod.rs b/cli/napi/mod.rs deleted file mode 100644 index 122d2ff0607f91..00000000000000 --- a/cli/napi/mod.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -#![allow(unused_mut)] -#![allow(non_camel_case_types)] -#![allow(clippy::undocumented_unsafe_blocks)] - -//! Symbols to be exported are now defined in this JSON file. -//! The `#[napi_sym]` macro checks for missing entries and panics. -//! -//! `./tools/napi/generate_symbols_list.js` is used to generate the LINK `cli/exports.def` on Windows, -//! which is also checked into git. -//! -//! To add a new napi function: -//! 1. Place `#[napi_sym]` on top of your implementation. -//! 2. Add the function's identifier to this JSON list. -//! 3. Finally, run `tools/napi/generate_symbols_list.js` to update `cli/napi/generated_symbol_exports_list_*.def`. - -pub mod js_native_api; -pub mod node_api; -pub mod util; diff --git a/cli/napi/sym/Cargo.toml b/cli/napi/sym/Cargo.toml deleted file mode 100644 index eba5fa41cf8507..00000000000000 --- a/cli/napi/sym/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -[package] -name = "napi_sym" -version = "0.98.0" -authors.workspace = true -edition.workspace = true -license.workspace = true -readme = "README.md" -repository.workspace = true -description = "proc macro for writing N-API symbols" - -[lib] -path = "./lib.rs" -proc-macro = true - -[dependencies] -quote.workspace = true -serde.workspace = true -serde_json.workspace = true -syn.workspace = true diff --git a/cli/napi/sym/README.md b/cli/napi/sym/README.md deleted file mode 100644 index de08a8e174eb20..00000000000000 --- a/cli/napi/sym/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# napi_sym - -A proc_macro for Deno's Node-API implementation. It does the following things: - -- Marks the symbol as `#[no_mangle]` and rewrites it as `pub extern "C" $name`. -- Asserts that the function symbol is present in - [`symbol_exports.json`](./symbol_exports.json). -- Maps `deno_napi::Result` to raw `napi_result`. - -```rust -use deno_napi::napi_value; -use deno_napi::Env; -use deno_napi::Error; -use deno_napi::Result; - -#[napi_sym::napi_sym] -fn napi_get_boolean( - env: *mut Env, - value: bool, - result: *mut napi_value, -) -> Result { - let _env: &mut Env = env.as_mut().ok_or(Error::InvalidArg)?; - // *result = ... - Ok(()) -} -``` - -### `symbol_exports.json` - -A file containing the symbols that need to be put into the executable's dynamic -symbol table at link-time. - -This is done using `/DEF:` on Windows, `-exported_symbol,_` on macOS and -`--export-dynamic-symbol=` on Linux. See [`cli/build.rs`](../build.rs). - -On Windows, you need to generate the `.def` file by running -[`tools/napi/generate_symbols_lists.js`](../../tools/napi/generate_symbols_lists.js). diff --git a/cli/node.rs b/cli/node.rs index a3cee8ddeeec3c..11959df6b9ba3c 100644 --- a/cli/node.rs +++ b/cli/node.rs @@ -1,10 +1,12 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::borrow::Cow; use std::sync::Arc; use deno_ast::MediaType; use deno_ast::ModuleSpecifier; use deno_core::error::AnyError; +use deno_graph::ParsedSourceStore; use deno_runtime::deno_fs; use deno_runtime::deno_node::DenoFsNodeResolverEnv; use node_resolver::analyze::CjsAnalysis as ExtNodeCjsAnalysis; @@ -16,8 +18,8 @@ use serde::Serialize; use crate::cache::CacheDBHash; use crate::cache::NodeAnalysisCache; -use crate::resolver::CliNodeResolver; -use crate::util::fs::canonicalize_path_maybe_not_exists; +use crate::cache::ParsedSourceCache; +use crate::resolver::CjsTracker; pub type CliNodeCodeTranslator = NodeCodeTranslator; @@ -30,15 +32,11 @@ pub type CliNodeCodeTranslator = /// because the node_modules folder might not exist at that time. pub fn resolve_specifier_into_node_modules( specifier: &ModuleSpecifier, + fs: &dyn deno_fs::FileSystem, ) -> ModuleSpecifier { - specifier - .to_file_path() - .ok() - // this path might not exist at the time the graph is being created - // because the node_modules folder might not yet exist - .and_then(|path| canonicalize_path_maybe_not_exists(&path).ok()) - .and_then(|path| ModuleSpecifier::from_file_path(path).ok()) - .unwrap_or_else(|| specifier.clone()) + node_resolver::resolve_specifier_into_node_modules(specifier, &|path| { + fs.realpath_sync(path).map_err(|err| err.into_io_error()) + }) } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -54,20 +52,23 @@ pub enum CliCjsAnalysis { pub struct CliCjsCodeAnalyzer { cache: NodeAnalysisCache, + cjs_tracker: Arc, fs: deno_fs::FileSystemRc, - node_resolver: Arc, + parsed_source_cache: Option>, } impl CliCjsCodeAnalyzer { pub fn new( cache: NodeAnalysisCache, + cjs_tracker: Arc, fs: deno_fs::FileSystemRc, - node_resolver: Arc, + parsed_source_cache: Option>, ) -> Self { Self { cache, + cjs_tracker, fs, - node_resolver, + parsed_source_cache, } } @@ -83,7 +84,7 @@ impl CliCjsCodeAnalyzer { return Ok(analysis); } - let mut media_type = MediaType::from_specifier(specifier); + let media_type = MediaType::from_specifier(specifier); if media_type == MediaType::Json { return Ok(CliCjsAnalysis::Cjs { exports: vec![], @@ -91,54 +92,51 @@ impl CliCjsCodeAnalyzer { }); } - if media_type == MediaType::JavaScript { - if let Some(package_json) = - self.node_resolver.get_closest_package_json(specifier)? - { - match package_json.typ.as_str() { - "commonjs" => { - media_type = MediaType::Cjs; - } - "module" => { - media_type = MediaType::Mjs; - } - _ => {} - } - } - } + let cjs_tracker = self.cjs_tracker.clone(); + let is_maybe_cjs = cjs_tracker.is_maybe_cjs(specifier, media_type)?; + let analysis = if is_maybe_cjs { + let maybe_parsed_source = self + .parsed_source_cache + .as_ref() + .and_then(|c| c.remove_parsed_source(specifier)); - let analysis = deno_core::unsync::spawn_blocking({ - let specifier = specifier.clone(); - let source: Arc = source.into(); - move || -> Result<_, deno_ast::ParseDiagnostic> { - let parsed_source = deno_ast::parse_program(deno_ast::ParseParams { - specifier, - text: source, - media_type, - capture_tokens: true, - scope_analysis: false, - maybe_syntax: None, - })?; - if parsed_source.is_script() { - let analysis = parsed_source.analyze_cjs(); - Ok(CliCjsAnalysis::Cjs { - exports: analysis.exports, - reexports: analysis.reexports, - }) - } else if media_type == MediaType::Cjs { - // FIXME: `deno_ast` should internally handle MediaType::Cjs implying that - // the result must never be Esm - Ok(CliCjsAnalysis::Cjs { - exports: vec![], - reexports: vec![], - }) - } else { - Ok(CliCjsAnalysis::Esm) + deno_core::unsync::spawn_blocking({ + let specifier = specifier.clone(); + let source: Arc = source.into(); + move || -> Result<_, AnyError> { + let parsed_source = + maybe_parsed_source.map(Ok).unwrap_or_else(|| { + deno_ast::parse_program(deno_ast::ParseParams { + specifier, + text: source, + media_type, + capture_tokens: true, + scope_analysis: false, + maybe_syntax: None, + }) + })?; + let is_script = parsed_source.compute_is_script(); + let is_cjs = cjs_tracker.is_cjs_with_known_is_script( + parsed_source.specifier(), + media_type, + is_script, + )?; + if is_cjs { + let analysis = parsed_source.analyze_cjs(); + Ok(CliCjsAnalysis::Cjs { + exports: analysis.exports, + reexports: analysis.reexports, + }) + } else { + Ok(CliCjsAnalysis::Esm) + } } - } - }) - .await - .unwrap()?; + }) + .await + .unwrap()? + } else { + CliCjsAnalysis::Esm + }; self .cache @@ -150,11 +148,11 @@ impl CliCjsCodeAnalyzer { #[async_trait::async_trait(?Send)] impl CjsCodeAnalyzer for CliCjsCodeAnalyzer { - async fn analyze_cjs( + async fn analyze_cjs<'a>( &self, specifier: &ModuleSpecifier, - source: Option, - ) -> Result { + source: Option>, + ) -> Result, AnyError> { let source = match source { Some(source) => source, None => { diff --git a/cli/npm/byonm.rs b/cli/npm/byonm.rs index fc7069e1f8179d..eca399251b0e31 100644 --- a/cli/npm/byonm.rs +++ b/cli/npm/byonm.rs @@ -2,393 +2,92 @@ use std::borrow::Cow; use std::path::Path; -use std::path::PathBuf; use std::sync::Arc; -use deno_ast::ModuleSpecifier; -use deno_core::anyhow::bail; use deno_core::error::AnyError; use deno_core::serde_json; -use deno_package_json::PackageJsonDepValue; -use deno_runtime::deno_fs::FileSystem; -use deno_runtime::deno_node::DenoPkgJsonFsAdapter; +use deno_resolver::npm::ByonmNpmResolver; +use deno_resolver::npm::ByonmNpmResolverCreateOptions; +use deno_resolver::npm::CliNpmReqResolver; +use deno_runtime::deno_node::DenoFsNodeResolverEnv; use deno_runtime::deno_node::NodePermissions; -use deno_runtime::deno_node::NodeRequireResolver; -use deno_runtime::deno_node::NpmProcessStateProvider; -use deno_runtime::deno_node::PackageJson; -use deno_runtime::fs_util::specifier_to_file_path; -use deno_semver::package::PackageReq; -use deno_semver::Version; -use node_resolver::errors::PackageFolderResolveError; -use node_resolver::errors::PackageFolderResolveIoError; -use node_resolver::errors::PackageJsonLoadError; -use node_resolver::errors::PackageNotFoundError; -use node_resolver::load_pkg_json; -use node_resolver::NpmResolver; +use deno_runtime::ops::process::NpmProcessStateProvider; +use node_resolver::NpmPackageFolderResolver; use crate::args::NpmProcessState; use crate::args::NpmProcessStateKind; -use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; +use crate::resolver::CliDenoResolverFs; -use super::managed::normalize_pkg_name_for_node_modules_deno_folder; use super::CliNpmResolver; use super::InnerCliNpmResolverRef; -pub struct CliNpmResolverByonmCreateOptions { - pub fs: Arc, - // todo(dsherret): investigate removing this - pub root_node_modules_dir: Option, -} - -pub fn create_byonm_npm_resolver( - options: CliNpmResolverByonmCreateOptions, -) -> Arc { - Arc::new(ByonmCliNpmResolver { - fs: options.fs, - root_node_modules_dir: options.root_node_modules_dir, - }) -} +pub type CliByonmNpmResolverCreateOptions = + ByonmNpmResolverCreateOptions; +pub type CliByonmNpmResolver = + ByonmNpmResolver; +// todo(dsherret): the services hanging off `CliNpmResolver` doesn't seem ideal. We should probably decouple. #[derive(Debug)] -pub struct ByonmCliNpmResolver { - fs: Arc, - root_node_modules_dir: Option, -} - -impl ByonmCliNpmResolver { - fn load_pkg_json( - &self, - path: &Path, - ) -> Result>, PackageJsonLoadError> { - load_pkg_json(&DenoPkgJsonFsAdapter(self.fs.as_ref()), path) - } - - /// Finds the ancestor package.json that contains the specified dependency. - pub fn find_ancestor_package_json_with_dep( - &self, - dep_name: &str, - referrer: &ModuleSpecifier, - ) -> Option> { - let referrer_path = referrer.to_file_path().ok()?; - let mut current_folder = referrer_path.parent()?; - loop { - let pkg_json_path = current_folder.join("package.json"); - if let Ok(Some(pkg_json)) = self.load_pkg_json(&pkg_json_path) { - if let Some(deps) = &pkg_json.dependencies { - if deps.contains_key(dep_name) { - return Some(pkg_json); - } - } - if let Some(deps) = &pkg_json.dev_dependencies { - if deps.contains_key(dep_name) { - return Some(pkg_json); - } - } - } - - if let Some(parent) = current_folder.parent() { - current_folder = parent; - } else { - return None; - } - } - } - - fn resolve_pkg_json_and_alias_for_req( - &self, - req: &PackageReq, - referrer: &ModuleSpecifier, - ) -> Result, String)>, AnyError> { - fn resolve_alias_from_pkg_json( - req: &PackageReq, - pkg_json: &PackageJson, - ) -> Option { - let deps = pkg_json.resolve_local_package_json_deps(); - for (key, value) in deps { - if let Ok(value) = value { - match value { - PackageJsonDepValue::Req(dep_req) => { - if dep_req.name == req.name - && dep_req.version_req.intersects(&req.version_req) - { - return Some(key); - } - } - PackageJsonDepValue::Workspace(_workspace) => { - if key == req.name && req.version_req.tag() == Some("workspace") { - return Some(key); - } - } - } - } - } - None - } - - // attempt to resolve the npm specifier from the referrer's package.json, - if let Ok(file_path) = specifier_to_file_path(referrer) { - let mut current_path = file_path.as_path(); - while let Some(dir_path) = current_path.parent() { - let package_json_path = dir_path.join("package.json"); - if let Some(pkg_json) = self.load_pkg_json(&package_json_path)? { - if let Some(alias) = - resolve_alias_from_pkg_json(req, pkg_json.as_ref()) - { - return Ok(Some((pkg_json, alias))); - } - } - current_path = dir_path; - } - } - - // otherwise, fall fallback to the project's package.json - if let Some(root_node_modules_dir) = &self.root_node_modules_dir { - let root_pkg_json_path = - root_node_modules_dir.parent().unwrap().join("package.json"); - if let Some(pkg_json) = self.load_pkg_json(&root_pkg_json_path)? { - if let Some(alias) = resolve_alias_from_pkg_json(req, pkg_json.as_ref()) - { - return Ok(Some((pkg_json, alias))); - } - } - } - - Ok(None) - } - - fn resolve_folder_in_root_node_modules( - &self, - req: &PackageReq, - ) -> Option { - // now check if node_modules/.deno/ matches this constraint - let root_node_modules_dir = self.root_node_modules_dir.as_ref()?; - let node_modules_deno_dir = root_node_modules_dir.join(".deno"); - let Ok(entries) = self.fs.read_dir_sync(&node_modules_deno_dir) else { - return None; - }; - let search_prefix = format!( - "{}@", - normalize_pkg_name_for_node_modules_deno_folder(&req.name) - ); - let mut best_version = None; - - // example entries: - // - @denotest+add@1.0.0 - // - @denotest+add@1.0.0_1 - for entry in entries { - if !entry.is_directory { - continue; - } - let Some(version_and_copy_idx) = entry.name.strip_prefix(&search_prefix) - else { - continue; - }; - let version = version_and_copy_idx - .rsplit_once('_') - .map(|(v, _)| v) - .unwrap_or(version_and_copy_idx); - let Ok(version) = Version::parse_from_npm(version) else { - continue; - }; - if req.version_req.matches(&version) { - if let Some((best_version_version, _)) = &best_version { - if version > *best_version_version { - best_version = Some((version, entry.name)); - } - } else { - best_version = Some((version, entry.name)); - } - } - } - - best_version.map(|(_version, entry_name)| { - join_package_name( - &node_modules_deno_dir.join(entry_name).join("node_modules"), - &req.name, - ) - }) - } -} - -impl NpmResolver for ByonmCliNpmResolver { - fn resolve_package_folder_from_package( - &self, - name: &str, - referrer: &ModuleSpecifier, - ) -> Result { - fn inner( - fs: &dyn FileSystem, - name: &str, - referrer: &ModuleSpecifier, - ) -> Result { - let maybe_referrer_file = specifier_to_file_path(referrer).ok(); - let maybe_start_folder = - maybe_referrer_file.as_ref().and_then(|f| f.parent()); - if let Some(start_folder) = maybe_start_folder { - for current_folder in start_folder.ancestors() { - let node_modules_folder = if current_folder.ends_with("node_modules") - { - Cow::Borrowed(current_folder) - } else { - Cow::Owned(current_folder.join("node_modules")) - }; - - let sub_dir = join_package_name(&node_modules_folder, name); - if fs.is_dir_sync(&sub_dir) { - return Ok(sub_dir); - } - } - } - - Err( - PackageNotFoundError { - package_name: name.to_string(), - referrer: referrer.clone(), - referrer_extra: None, - } - .into(), - ) - } +struct CliByonmWrapper(Arc); - let path = inner(&*self.fs, name, referrer)?; - self.fs.realpath_sync(&path).map_err(|err| { - PackageFolderResolveIoError { - package_name: name.to_string(), - referrer: referrer.clone(), - source: err.into_io_error(), - } - .into() - }) - } - - fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool { - specifier.scheme() == "file" - && specifier - .path() - .to_ascii_lowercase() - .contains("/node_modules/") - } -} - -impl NodeRequireResolver for ByonmCliNpmResolver { - fn ensure_read_permission( - &self, - permissions: &mut dyn NodePermissions, - path: &Path, - ) -> Result<(), AnyError> { - if !path - .components() - .any(|c| c.as_os_str().to_ascii_lowercase() == "node_modules") - { - _ = permissions.check_read_path(path)?; - } - Ok(()) - } -} - -impl NpmProcessStateProvider for ByonmCliNpmResolver { +impl NpmProcessStateProvider for CliByonmWrapper { fn get_npm_process_state(&self) -> String { serde_json::to_string(&NpmProcessState { kind: NpmProcessStateKind::Byonm, local_node_modules_path: self - .root_node_modules_dir - .as_ref() + .0 + .root_node_modules_dir() .map(|p| p.to_string_lossy().to_string()), }) .unwrap() } } -impl CliNpmResolver for ByonmCliNpmResolver { - fn into_npm_resolver(self: Arc) -> Arc { +impl CliNpmResolver for CliByonmNpmResolver { + fn into_npm_pkg_folder_resolver( + self: Arc, + ) -> Arc { self } - fn into_require_resolver(self: Arc) -> Arc { + fn into_npm_req_resolver(self: Arc) -> Arc { self } fn into_process_state_provider( self: Arc, ) -> Arc { - self + Arc::new(CliByonmWrapper(self)) + } + + fn into_maybe_byonm(self: Arc) -> Option> { + Some(self) } fn clone_snapshotted(&self) -> Arc { - Arc::new(Self { - fs: self.fs.clone(), - root_node_modules_dir: self.root_node_modules_dir.clone(), - }) + Arc::new(self.clone()) } fn as_inner(&self) -> InnerCliNpmResolverRef { InnerCliNpmResolverRef::Byonm(self) } - fn root_node_modules_path(&self) -> Option<&PathBuf> { - self.root_node_modules_dir.as_ref() + fn root_node_modules_path(&self) -> Option<&Path> { + self.root_node_modules_dir() } - fn resolve_pkg_folder_from_deno_module_req( + fn ensure_read_permission<'a>( &self, - req: &PackageReq, - referrer: &ModuleSpecifier, - ) -> Result { - fn node_resolve_dir( - fs: &dyn FileSystem, - alias: &str, - start_dir: &Path, - ) -> Result, AnyError> { - for ancestor in start_dir.ancestors() { - let node_modules_folder = ancestor.join("node_modules"); - let sub_dir = join_package_name(&node_modules_folder, alias); - if fs.is_dir_sync(&sub_dir) { - return Ok(Some(canonicalize_path_maybe_not_exists_with_fs( - &sub_dir, fs, - )?)); - } - } - Ok(None) - } - - // now attempt to resolve if it's found in any package.json - let maybe_pkg_json_and_alias = - self.resolve_pkg_json_and_alias_for_req(req, referrer)?; - match maybe_pkg_json_and_alias { - Some((pkg_json, alias)) => { - // now try node resolution - if let Some(resolved) = - node_resolve_dir(self.fs.as_ref(), &alias, pkg_json.dir_path())? - { - return Ok(resolved); - } - - bail!( - concat!( - "Could not find \"{}\" in a node_modules folder. ", - "Deno expects the node_modules/ directory to be up to date. ", - "Did you forget to run `deno install`?" - ), - alias, - ); - } - None => { - // now check if node_modules/.deno/ matches this constraint - if let Some(folder) = self.resolve_folder_in_root_node_modules(req) { - return Ok(folder); - } - - bail!( - concat!( - "Could not find a matching package for 'npm:{}' in the node_modules ", - "directory. Ensure you have all your JSR and npm dependencies listed ", - "in your deno.json or package.json, then run `deno install`. Alternatively, ", - r#"turn on auto-install by specifying `"nodeModulesDir": "auto"` in your "#, - "deno.json file." - ), - req, - ); - } + permissions: &mut dyn NodePermissions, + path: &'a Path, + ) -> Result, AnyError> { + if !path + .components() + .any(|c| c.as_os_str().to_ascii_lowercase() == "node_modules") + { + permissions.check_read_path(path).map_err(Into::into) + } else { + Ok(Cow::Borrowed(path)) } } @@ -398,12 +97,3 @@ impl CliNpmResolver for ByonmCliNpmResolver { None } } - -fn join_package_name(path: &Path, package_name: &str) -> PathBuf { - let mut path = path.to_path_buf(); - // ensure backslashes are used on windows - for part in package_name.split('/') { - path = path.join(part); - } - path -} diff --git a/cli/npm/cache_dir.rs b/cli/npm/cache_dir.rs deleted file mode 100644 index 4467d685ec520b..00000000000000 --- a/cli/npm/cache_dir.rs +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -use std::path::Path; -use std::path::PathBuf; - -use deno_ast::ModuleSpecifier; -use deno_core::anyhow::Context; -use deno_core::error::AnyError; -use deno_core::url::Url; -use deno_npm::NpmPackageCacheFolderId; -use deno_semver::package::PackageNv; -use deno_semver::Version; - -use crate::util::fs::canonicalize_path; -use crate::util::path::root_url_to_safe_local_dirname; - -/// The global cache directory of npm packages. -#[derive(Clone, Debug)] -pub struct NpmCacheDir { - root_dir: PathBuf, - // cached url representation of the root directory - root_dir_url: Url, - // A list of all registry that were discovered via `.npmrc` files - // turned into a safe directory names. - known_registries_dirnames: Vec, -} - -impl NpmCacheDir { - pub fn new(root_dir: PathBuf, known_registries_urls: Vec) -> Self { - fn try_get_canonicalized_root_dir( - root_dir: &Path, - ) -> Result { - if !root_dir.exists() { - std::fs::create_dir_all(root_dir) - .with_context(|| format!("Error creating {}", root_dir.display()))?; - } - Ok(canonicalize_path(root_dir)?) - } - - // this may fail on readonly file systems, so just ignore if so - let root_dir = - try_get_canonicalized_root_dir(&root_dir).unwrap_or(root_dir); - let root_dir_url = Url::from_directory_path(&root_dir).unwrap(); - - let known_registries_dirnames: Vec<_> = known_registries_urls - .into_iter() - .map(|url| { - root_url_to_safe_local_dirname(&url) - .to_string_lossy() - .replace('\\', "/") - }) - .collect(); - - Self { - root_dir, - root_dir_url, - known_registries_dirnames, - } - } - - pub fn root_dir(&self) -> &Path { - &self.root_dir - } - - pub fn root_dir_url(&self) -> &Url { - &self.root_dir_url - } - - pub fn package_folder_for_id( - &self, - folder_id: &NpmPackageCacheFolderId, - registry_url: &Url, - ) -> PathBuf { - if folder_id.copy_index == 0 { - self.package_folder_for_nv(&folder_id.nv, registry_url) - } else { - self - .package_name_folder(&folder_id.nv.name, registry_url) - .join(format!("{}_{}", folder_id.nv.version, folder_id.copy_index)) - } - } - - pub fn package_folder_for_nv( - &self, - package: &PackageNv, - registry_url: &Url, - ) -> PathBuf { - self - .package_name_folder(&package.name, registry_url) - .join(package.version.to_string()) - } - - pub fn package_name_folder(&self, name: &str, registry_url: &Url) -> PathBuf { - let mut dir = self.registry_folder(registry_url); - if name.to_lowercase() != name { - let encoded_name = mixed_case_package_name_encode(name); - // Using the encoded directory may have a collision with an actual package name - // so prefix it with an underscore since npm packages can't start with that - dir.join(format!("_{encoded_name}")) - } else { - // ensure backslashes are used on windows - for part in name.split('/') { - dir = dir.join(part); - } - dir - } - } - - fn registry_folder(&self, registry_url: &Url) -> PathBuf { - self - .root_dir - .join(root_url_to_safe_local_dirname(registry_url)) - } - - pub fn resolve_package_folder_id_from_specifier( - &self, - specifier: &ModuleSpecifier, - ) -> Option { - let mut maybe_relative_url = None; - - // Iterate through known registries and try to get a match. - for registry_dirname in &self.known_registries_dirnames { - let registry_root_dir = self - .root_dir_url - .join(&format!("{}/", registry_dirname)) - // this not succeeding indicates a fatal issue, so unwrap - .unwrap(); - - let Some(relative_url) = registry_root_dir.make_relative(specifier) - else { - continue; - }; - - if relative_url.starts_with("../") { - continue; - } - - maybe_relative_url = Some(relative_url); - break; - } - - let mut relative_url = maybe_relative_url?; - - // base32 decode the url if it starts with an underscore - // * Ex. _{base32(package_name)}/ - if let Some(end_url) = relative_url.strip_prefix('_') { - let mut parts = end_url - .split('/') - .map(ToOwned::to_owned) - .collect::>(); - match mixed_case_package_name_decode(&parts[0]) { - Some(part) => { - parts[0] = part; - } - None => return None, - } - relative_url = parts.join("/"); - } - - // examples: - // * chalk/5.0.1/ - // * @types/chalk/5.0.1/ - // * some-package/5.0.1_1/ -- where the `_1` (/_\d+/) is a copy of the folder for peer deps - let is_scoped_package = relative_url.starts_with('@'); - let mut parts = relative_url - .split('/') - .enumerate() - .take(if is_scoped_package { 3 } else { 2 }) - .map(|(_, part)| part) - .collect::>(); - if parts.len() < 2 { - return None; - } - let version_part = parts.pop().unwrap(); - let name = parts.join("/"); - let (version, copy_index) = - if let Some((version, copy_count)) = version_part.split_once('_') { - (version, copy_count.parse::().ok()?) - } else { - (version_part, 0) - }; - Some(NpmPackageCacheFolderId { - nv: PackageNv { - name, - version: Version::parse_from_npm(version).ok()?, - }, - copy_index, - }) - } - - pub fn get_cache_location(&self) -> PathBuf { - self.root_dir.clone() - } -} - -pub fn mixed_case_package_name_encode(name: &str) -> String { - // use base32 encoding because it's reversible and the character set - // only includes the characters within 0-9 and A-Z so it can be lower cased - base32::encode( - base32::Alphabet::Rfc4648Lower { padding: false }, - name.as_bytes(), - ) - .to_lowercase() -} - -pub fn mixed_case_package_name_decode(name: &str) -> Option { - base32::decode(base32::Alphabet::Rfc4648Lower { padding: false }, name) - .and_then(|b| String::from_utf8(b).ok()) -} - -#[cfg(test)] -mod test { - use deno_core::url::Url; - use deno_semver::package::PackageNv; - use deno_semver::Version; - - use super::NpmCacheDir; - use crate::npm::cache_dir::NpmPackageCacheFolderId; - - #[test] - fn should_get_package_folder() { - let deno_dir = crate::cache::DenoDir::new(None).unwrap(); - let root_dir = deno_dir.npm_folder_path(); - let registry_url = Url::parse("https://registry.npmjs.org/").unwrap(); - let cache = NpmCacheDir::new(root_dir.clone(), vec![registry_url.clone()]); - - assert_eq!( - cache.package_folder_for_id( - &NpmPackageCacheFolderId { - nv: PackageNv { - name: "json".to_string(), - version: Version::parse_from_npm("1.2.5").unwrap(), - }, - copy_index: 0, - }, - ®istry_url, - ), - root_dir - .join("registry.npmjs.org") - .join("json") - .join("1.2.5"), - ); - - assert_eq!( - cache.package_folder_for_id( - &NpmPackageCacheFolderId { - nv: PackageNv { - name: "json".to_string(), - version: Version::parse_from_npm("1.2.5").unwrap(), - }, - copy_index: 1, - }, - ®istry_url, - ), - root_dir - .join("registry.npmjs.org") - .join("json") - .join("1.2.5_1"), - ); - - assert_eq!( - cache.package_folder_for_id( - &NpmPackageCacheFolderId { - nv: PackageNv { - name: "JSON".to_string(), - version: Version::parse_from_npm("2.1.5").unwrap(), - }, - copy_index: 0, - }, - ®istry_url, - ), - root_dir - .join("registry.npmjs.org") - .join("_jjju6tq") - .join("2.1.5"), - ); - - assert_eq!( - cache.package_folder_for_id( - &NpmPackageCacheFolderId { - nv: PackageNv { - name: "@types/JSON".to_string(), - version: Version::parse_from_npm("2.1.5").unwrap(), - }, - copy_index: 0, - }, - ®istry_url, - ), - root_dir - .join("registry.npmjs.org") - .join("_ib2hs4dfomxuuu2pjy") - .join("2.1.5"), - ); - } -} diff --git a/cli/npm/common.rs b/cli/npm/common.rs index a3a828e745b380..55f1bc086dbb3e 100644 --- a/cli/npm/common.rs +++ b/cli/npm/common.rs @@ -3,6 +3,7 @@ use base64::prelude::BASE64_STANDARD; use base64::Engine; use deno_core::anyhow::bail; +use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_npm::npm_rc::RegistryConfig; use http::header; @@ -36,17 +37,21 @@ pub fn maybe_auth_header_for_npm_registry( } if username.is_some() && password.is_some() { + // The npm client does some double encoding when generating the + // bearer token value, see + // https://github.com/npm/cli/blob/780afc50e3a345feb1871a28e33fa48235bc3bd5/workspaces/config/lib/index.js#L846-L851 + let pw_base64 = BASE64_STANDARD + .decode(password.unwrap()) + .with_context(|| "The password in npmrc is an invalid base64 string")?; + let bearer = BASE64_STANDARD.encode(format!( + "{}:{}", + username.unwrap(), + String::from_utf8_lossy(&pw_base64) + )); + return Ok(Some(( header::AUTHORIZATION, - header::HeaderValue::from_str(&format!( - "Basic {}", - BASE64_STANDARD.encode(&format!( - "{}:{}", - username.unwrap(), - password.unwrap() - )) - )) - .unwrap(), + header::HeaderValue::from_str(&format!("Basic {}", bearer)).unwrap(), ))); } diff --git a/cli/npm/managed/cache/mod.rs b/cli/npm/managed/cache/mod.rs index f409744b985fbe..8ae99f41e014b5 100644 --- a/cli/npm/managed/cache/mod.rs +++ b/cli/npm/managed/cache/mod.rs @@ -8,6 +8,7 @@ use std::path::PathBuf; use std::sync::Arc; use deno_ast::ModuleSpecifier; +use deno_cache_dir::npm::NpmCacheDir; use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; @@ -18,14 +19,14 @@ use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::registry::NpmPackageInfo; use deno_npm::NpmPackageCacheFolderId; use deno_semver::package::PackageNv; +use deno_semver::Version; use crate::args::CacheSetting; use crate::cache::CACHE_PERM; -use crate::npm::NpmCacheDir; use crate::util::fs::atomic_write_file_with_retries; use crate::util::fs::hard_link_dir_recursive; -mod registry_info; +pub mod registry_info; mod tarball; mod tarball_extract; @@ -35,7 +36,7 @@ pub use tarball::TarballCache; /// Stores a single copy of npm packages in a cache. #[derive(Debug)] pub struct NpmCache { - cache_dir: NpmCacheDir, + cache_dir: Arc, cache_setting: CacheSetting, npmrc: Arc, /// ensures a package is only downloaded once per run @@ -44,7 +45,7 @@ pub struct NpmCache { impl NpmCache { pub fn new( - cache_dir: NpmCacheDir, + cache_dir: Arc, cache_setting: CacheSetting, npmrc: Arc, ) -> Self { @@ -60,6 +61,10 @@ impl NpmCache { &self.cache_setting } + pub fn root_dir_path(&self) -> &Path { + self.cache_dir.root_dir() + } + pub fn root_dir_url(&self) -> &Url { self.cache_dir.root_dir_url() } @@ -87,9 +92,12 @@ impl NpmCache { ) -> Result<(), AnyError> { let registry_url = self.npmrc.get_registry_url(&folder_id.nv.name); assert_ne!(folder_id.copy_index, 0); - let package_folder = self - .cache_dir - .package_folder_for_id(folder_id, registry_url); + let package_folder = self.cache_dir.package_folder_for_id( + &folder_id.nv.name, + &folder_id.nv.version.to_string(), + folder_id.copy_index, + registry_url, + ); if package_folder.exists() // if this file exists, then the package didn't successfully initialize @@ -100,9 +108,12 @@ impl NpmCache { return Ok(()); } - let original_package_folder = self - .cache_dir - .package_folder_for_nv(&folder_id.nv, registry_url); + let original_package_folder = self.cache_dir.package_folder_for_id( + &folder_id.nv.name, + &folder_id.nv.version.to_string(), + 0, // original copy index + registry_url, + ); // it seems Windows does an "AccessDenied" error when moving a // directory with hard links, so that's why this solution is done @@ -114,7 +125,12 @@ impl NpmCache { pub fn package_folder_for_id(&self, id: &NpmPackageCacheFolderId) -> PathBuf { let registry_url = self.npmrc.get_registry_url(&id.nv.name); - self.cache_dir.package_folder_for_id(id, registry_url) + self.cache_dir.package_folder_for_id( + &id.nv.name, + &id.nv.version.to_string(), + id.copy_index, + registry_url, + ) } pub fn package_folder_for_nv(&self, package: &PackageNv) -> PathBuf { @@ -127,7 +143,12 @@ impl NpmCache { package: &PackageNv, registry_url: &Url, ) -> PathBuf { - self.cache_dir.package_folder_for_nv(package, registry_url) + self.cache_dir.package_folder_for_id( + &package.name, + &package.version.to_string(), + 0, // original copy_index + registry_url, + ) } pub fn package_name_folder(&self, name: &str) -> PathBuf { @@ -135,10 +156,6 @@ impl NpmCache { self.cache_dir.package_name_folder(name, registry_url) } - pub fn root_folder(&self) -> PathBuf { - self.cache_dir.root_dir().to_owned() - } - pub fn resolve_package_folder_id_from_specifier( &self, specifier: &ModuleSpecifier, @@ -146,6 +163,15 @@ impl NpmCache { self .cache_dir .resolve_package_folder_id_from_specifier(specifier) + .and_then(|cache_id| { + Some(NpmPackageCacheFolderId { + nv: PackageNv { + name: cache_id.name, + version: Version::parse_from_npm(&cache_id.version).ok()?, + }, + copy_index: cache_id.copy_index, + }) + }) } pub fn load_package_info( diff --git a/cli/npm/managed/cache/registry_info.rs b/cli/npm/managed/cache/registry_info.rs index 28b19373e918e0..6d39d3c13f88c0 100644 --- a/cli/npm/managed/cache/registry_info.rs +++ b/cli/npm/managed/cache/registry_info.rs @@ -84,7 +84,7 @@ impl RegistryInfoDownloader { self.load_package_info_inner(name).await.with_context(|| { format!( "Error getting response at {} for package \"{}\"", - self.get_package_url(name), + get_package_url(&self.npmrc, name), name ) }) @@ -190,7 +190,7 @@ impl RegistryInfoDownloader { fn create_load_future(self: &Arc, name: &str) -> LoadFuture { let downloader = self.clone(); - let package_url = self.get_package_url(name); + let package_url = get_package_url(&self.npmrc, name); let registry_config = self.npmrc.get_registry_config(name); let maybe_auth_header = match maybe_auth_header_for_npm_registry(registry_config) { @@ -202,10 +202,13 @@ impl RegistryInfoDownloader { let guard = self.progress_bar.update(package_url.as_str()); let name = name.to_string(); async move { - let maybe_bytes = downloader - .http_client_provider - .get_or_create()? - .download_with_progress(package_url, maybe_auth_header, &guard) + let client = downloader.http_client_provider.get_or_create()?; + let maybe_bytes = client + .download_with_progress_and_retries( + package_url, + maybe_auth_header, + &guard, + ) .await?; match maybe_bytes { Some(bytes) => { @@ -236,25 +239,36 @@ impl RegistryInfoDownloader { .map(|r| r.map_err(Arc::new)) .boxed_local() } +} - fn get_package_url(&self, name: &str) -> Url { - let registry_url = self.npmrc.get_registry_url(name); - // list of all characters used in npm packages: - // !, ', (, ), *, -, ., /, [0-9], @, [A-Za-z], _, ~ - const ASCII_SET: percent_encoding::AsciiSet = - percent_encoding::NON_ALPHANUMERIC - .remove(b'!') - .remove(b'\'') - .remove(b'(') - .remove(b')') - .remove(b'*') - .remove(b'-') - .remove(b'.') - .remove(b'/') - .remove(b'@') - .remove(b'_') - .remove(b'~'); - let name = percent_encoding::utf8_percent_encode(name, &ASCII_SET); - registry_url.join(&name.to_string()).unwrap() - } +pub fn get_package_url(npmrc: &ResolvedNpmRc, name: &str) -> Url { + let registry_url = npmrc.get_registry_url(name); + // The '/' character in scoped package names "@scope/name" must be + // encoded for older third party registries. Newer registries and + // npm itself support both ways + // - encoded: https://registry.npmjs.org/@rollup%2fplugin-json + // - non-ecoded: https://registry.npmjs.org/@rollup/plugin-json + // To support as many third party registries as possible we'll + // always encode the '/' character. + + // list of all characters used in npm packages: + // !, ', (, ), *, -, ., /, [0-9], @, [A-Za-z], _, ~ + const ASCII_SET: percent_encoding::AsciiSet = + percent_encoding::NON_ALPHANUMERIC + .remove(b'!') + .remove(b'\'') + .remove(b'(') + .remove(b')') + .remove(b'*') + .remove(b'-') + .remove(b'.') + .remove(b'@') + .remove(b'_') + .remove(b'~'); + let name = percent_encoding::utf8_percent_encode(name, &ASCII_SET); + registry_url + // Ensure that scoped package name percent encoding is lower cased + // to match npm. + .join(&name.to_string().replace("%2F", "%2f")) + .unwrap() } diff --git a/cli/npm/managed/cache/tarball.rs b/cli/npm/managed/cache/tarball.rs index 4bcee38ea85d8c..7cf88d6d64d54e 100644 --- a/cli/npm/managed/cache/tarball.rs +++ b/cli/npm/managed/cache/tarball.rs @@ -172,7 +172,7 @@ impl TarballCache { let guard = tarball_cache.progress_bar.update(&dist.tarball); let result = tarball_cache.http_client_provider .get_or_create()? - .download_with_progress(tarball_uri, maybe_auth_header, &guard) + .download_with_progress_and_retries(tarball_uri, maybe_auth_header, &guard) .await; let maybe_bytes = match result { Ok(maybe_bytes) => maybe_bytes, diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs index 0be26b785779fb..88094d51414f74 100644 --- a/cli/npm/managed/mod.rs +++ b/cli/npm/managed/mod.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::borrow::Cow; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -7,9 +8,11 @@ use std::sync::Arc; use cache::RegistryInfoDownloader; use cache::TarballCache; use deno_ast::ModuleSpecifier; +use deno_cache_dir::npm::NpmCacheDir; use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::serde_json; +use deno_core::url::Url; use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::registry::NpmPackageInfo; use deno_npm::registry::NpmRegistryApi; @@ -19,15 +22,17 @@ use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmPackageId; use deno_npm::NpmResolutionPackage; use deno_npm::NpmSystemInfo; +use deno_resolver::npm::CliNpmReqResolver; +use deno_runtime::colors; use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_node::NodePermissions; -use deno_runtime::deno_node::NodeRequireResolver; -use deno_runtime::deno_node::NpmProcessStateProvider; +use deno_runtime::ops::process::NpmProcessStateProvider; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; use node_resolver::errors::PackageFolderResolveError; use node_resolver::errors::PackageFolderResolveIoError; -use node_resolver::NpmResolver; +use node_resolver::InNpmPackageChecker; +use node_resolver::NpmPackageFolderResolver; use resolution::AddPkgReqsResult; use crate::args::CliLockfile; @@ -35,6 +40,7 @@ use crate::args::LifecycleScriptsConfig; use crate::args::NpmInstallDepsProvider; use crate::args::NpmProcessState; use crate::args::NpmProcessStateKind; +use crate::args::PackageJsonDepValueParseWithLocationError; use crate::cache::FastInsecureHasher; use crate::http_util::HttpClientProvider; use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; @@ -45,14 +51,13 @@ use self::cache::NpmCache; use self::registry::CliNpmRegistryApi; use self::resolution::NpmResolution; use self::resolvers::create_npm_fs_resolver; -pub use self::resolvers::normalize_pkg_name_for_node_modules_deno_folder; use self::resolvers::NpmPackageFsResolver; use super::CliNpmResolver; use super::InnerCliNpmResolverRef; -use super::NpmCacheDir; +use super::ResolvePkgFolderFromDenoReqError; -mod cache; +pub mod cache; mod registry; mod resolution; mod resolvers; @@ -62,12 +67,12 @@ pub enum CliNpmResolverManagedSnapshotOption { Specified(Option), } -pub struct CliNpmResolverManagedCreateOptions { +pub struct CliManagedNpmResolverCreateOptions { pub snapshot: CliNpmResolverManagedSnapshotOption, pub maybe_lockfile: Option>, pub fs: Arc, pub http_client_provider: Arc, - pub npm_global_cache_dir: PathBuf, + pub npm_cache_dir: Arc, pub cache_setting: crate::args::CacheSetting, pub text_only_progress_bar: crate::util::progress_bar::ProgressBar, pub maybe_node_modules_path: Option, @@ -78,7 +83,7 @@ pub struct CliNpmResolverManagedCreateOptions { } pub async fn create_managed_npm_resolver_for_lsp( - options: CliNpmResolverManagedCreateOptions, + options: CliManagedNpmResolverCreateOptions, ) -> Arc { let npm_cache = create_cache(&options); let npm_api = create_api(&options, npm_cache.clone()); @@ -111,7 +116,7 @@ pub async fn create_managed_npm_resolver_for_lsp( } pub async fn create_managed_npm_resolver( - options: CliNpmResolverManagedCreateOptions, + options: CliManagedNpmResolverCreateOptions, ) -> Result, AnyError> { let npm_cache = create_cache(&options); let npm_api = create_api(&options, npm_cache.clone()); @@ -185,19 +190,16 @@ fn create_inner( )) } -fn create_cache(options: &CliNpmResolverManagedCreateOptions) -> Arc { +fn create_cache(options: &CliManagedNpmResolverCreateOptions) -> Arc { Arc::new(NpmCache::new( - NpmCacheDir::new( - options.npm_global_cache_dir.clone(), - options.npmrc.get_all_known_registries_urls(), - ), + options.npm_cache_dir.clone(), options.cache_setting.clone(), options.npmrc.clone(), )) } fn create_api( - options: &CliNpmResolverManagedCreateOptions, + options: &CliManagedNpmResolverCreateOptions, npm_cache: Arc, ) -> Arc { Arc::new(CliNpmRegistryApi::new( @@ -254,6 +256,35 @@ async fn snapshot_from_lockfile( Ok(snapshot) } +#[derive(Debug)] +struct ManagedInNpmPackageChecker { + root_dir: Url, +} + +impl InNpmPackageChecker for ManagedInNpmPackageChecker { + fn in_npm_package(&self, specifier: &Url) -> bool { + specifier.as_ref().starts_with(self.root_dir.as_str()) + } +} + +pub struct CliManagedInNpmPkgCheckerCreateOptions<'a> { + pub root_cache_dir_url: &'a Url, + pub maybe_node_modules_path: Option<&'a Path>, +} + +pub fn create_managed_in_npm_pkg_checker( + options: CliManagedInNpmPkgCheckerCreateOptions, +) -> Arc { + let root_dir = match options.maybe_node_modules_path { + Some(node_modules_folder) => { + deno_path_util::url_from_directory_path(node_modules_folder).unwrap() + } + None => options.root_cache_dir_url.clone(), + }; + debug_assert!(root_dir.as_str().ends_with('/')); + Arc::new(ManagedInNpmPackageChecker { root_dir }) +} + /// An npm resolver where the resolution is managed by Deno rather than /// the user bringing their own node_modules (BYONM) on the file system. pub struct ManagedCliNpmResolver { @@ -427,6 +458,16 @@ impl ManagedCliNpmResolver { self.resolution.snapshot() } + pub fn top_package_req_for_name(&self, name: &str) -> Option { + let package_reqs = self.resolution.package_reqs(); + let mut entries = package_reqs + .iter() + .filter(|(_, nv)| nv.name == name) + .collect::>(); + entries.sort_by_key(|(_, nv)| &nv.version); + Some(entries.last()?.0.clone()) + } + pub fn serialized_valid_snapshot_for_system( &self, system_info: &NpmSystemInfo, @@ -459,13 +500,37 @@ impl ManagedCliNpmResolver { self.resolve_pkg_folder_from_pkg_id(&pkg_id) } - fn resolve_pkg_id_from_pkg_req( + pub fn resolve_pkg_id_from_pkg_req( &self, req: &PackageReq, ) -> Result { self.resolution.resolve_pkg_id_from_pkg_req(req) } + pub fn ensure_no_pkg_json_dep_errors( + &self, + ) -> Result<(), Box> { + for err in self.npm_install_deps_provider.pkg_json_dep_errors() { + match &err.source { + deno_package_json::PackageJsonDepValueParseError::VersionReq(_) => { + return Err(Box::new(err.clone())); + } + deno_package_json::PackageJsonDepValueParseError::Unsupported { + .. + } => { + // only warn for this one + log::warn!( + "{} {}\n at {}", + colors::yellow("Warning"), + err.source, + err.location, + ) + } + } + } + Ok(()) + } + /// Ensures that the top level `package.json` dependencies are installed. /// This may set up the `node_modules` directory. /// @@ -477,6 +542,7 @@ impl ManagedCliNpmResolver { if !self.top_level_install_flag.raise() { return Ok(false); // already did this } + let pkg_json_remote_pkgs = self.npm_install_deps_provider.remote_pkgs(); if pkg_json_remote_pkgs.is_empty() { return Ok(false); @@ -515,8 +581,16 @@ impl ManagedCliNpmResolver { .map_err(|err| err.into()) } - pub fn global_cache_root_folder(&self) -> PathBuf { - self.npm_cache.root_folder() + pub fn maybe_node_modules_path(&self) -> Option<&Path> { + self.fs_resolver.node_modules_path() + } + + pub fn global_cache_root_path(&self) -> &Path { + self.npm_cache.root_dir_path() + } + + pub fn global_cache_root_url(&self) -> &Url { + self.npm_cache.root_dir_url() } } @@ -532,7 +606,7 @@ fn npm_process_state( .unwrap() } -impl NpmResolver for ManagedCliNpmResolver { +impl NpmPackageFolderResolver for ManagedCliNpmResolver { fn resolve_package_folder_from_package( &self, name: &str, @@ -551,39 +625,40 @@ impl NpmResolver for ManagedCliNpmResolver { log::debug!("Resolved {} from {} to {}", name, referrer, path.display()); Ok(path) } - - fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool { - let root_dir_url = self.fs_resolver.root_dir_url(); - debug_assert!(root_dir_url.as_str().ends_with('/')); - specifier.as_ref().starts_with(root_dir_url.as_str()) - } -} - -impl NodeRequireResolver for ManagedCliNpmResolver { - fn ensure_read_permission( - &self, - permissions: &mut dyn NodePermissions, - path: &Path, - ) -> Result<(), AnyError> { - self.fs_resolver.ensure_read_permission(permissions, path) - } } impl NpmProcessStateProvider for ManagedCliNpmResolver { fn get_npm_process_state(&self) -> String { npm_process_state( self.resolution.serialized_valid_snapshot(), - self.fs_resolver.node_modules_path().map(|p| p.as_path()), + self.fs_resolver.node_modules_path(), ) } } +impl CliNpmReqResolver for ManagedCliNpmResolver { + fn resolve_pkg_folder_from_deno_module_req( + &self, + req: &PackageReq, + _referrer: &ModuleSpecifier, + ) -> Result { + let pkg_id = self + .resolve_pkg_id_from_pkg_req(req) + .map_err(|err| ResolvePkgFolderFromDenoReqError::Managed(err.into()))?; + self + .resolve_pkg_folder_from_pkg_id(&pkg_id) + .map_err(ResolvePkgFolderFromDenoReqError::Managed) + } +} + impl CliNpmResolver for ManagedCliNpmResolver { - fn into_npm_resolver(self: Arc) -> Arc { + fn into_npm_pkg_folder_resolver( + self: Arc, + ) -> Arc { self } - fn into_require_resolver(self: Arc) -> Arc { + fn into_npm_req_resolver(self: Arc) -> Arc { self } @@ -630,17 +705,16 @@ impl CliNpmResolver for ManagedCliNpmResolver { InnerCliNpmResolverRef::Managed(self) } - fn root_node_modules_path(&self) -> Option<&PathBuf> { + fn root_node_modules_path(&self) -> Option<&Path> { self.fs_resolver.node_modules_path() } - fn resolve_pkg_folder_from_deno_module_req( + fn ensure_read_permission<'a>( &self, - req: &PackageReq, - _referrer: &ModuleSpecifier, - ) -> Result { - let pkg_id = self.resolve_pkg_id_from_pkg_req(req)?; - self.resolve_pkg_folder_from_pkg_id(&pkg_id) + permissions: &mut dyn NodePermissions, + path: &'a Path, + ) -> Result, AnyError> { + self.fs_resolver.ensure_read_permission(permissions, path) } fn check_state_hash(&self) -> Option { diff --git a/cli/npm/managed/resolvers/common.rs b/cli/npm/managed/resolvers/common.rs index 620daf4b30b4ab..eee11c7604952f 100644 --- a/cli/npm/managed/resolvers/common.rs +++ b/cli/npm/managed/resolvers/common.rs @@ -3,6 +3,7 @@ pub mod bin_entries; pub mod lifecycle_scripts; +use std::borrow::Cow; use std::collections::HashMap; use std::io::ErrorKind; use std::path::Path; @@ -16,7 +17,6 @@ use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::futures; use deno_core::futures::StreamExt; -use deno_core::url::Url; use deno_npm::NpmPackageCacheFolderId; use deno_npm::NpmPackageId; use deno_npm::NpmResolutionPackage; @@ -29,11 +29,8 @@ use crate::npm::managed::cache::TarballCache; /// Part of the resolution that interacts with the file system. #[async_trait(?Send)] pub trait NpmPackageFsResolver: Send + Sync { - /// Specifier for the root directory. - fn root_dir_url(&self) -> &Url; - /// The local node_modules folder if it is applicable to the implementation. - fn node_modules_path(&self) -> Option<&PathBuf>; + fn node_modules_path(&self) -> Option<&Path>; fn maybe_package_folder(&self, package_id: &NpmPackageId) -> Option; @@ -62,11 +59,12 @@ pub trait NpmPackageFsResolver: Send + Sync { async fn cache_packages(&self) -> Result<(), AnyError>; - fn ensure_read_permission( + #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] + fn ensure_read_permission<'a>( &self, permissions: &mut dyn NodePermissions, - path: &Path, - ) -> Result<(), AnyError>; + path: &'a Path, + ) -> Result, AnyError>; } #[derive(Debug)] @@ -85,11 +83,15 @@ impl RegistryReadPermissionChecker { } } - pub fn ensure_registry_read_permission( + pub fn ensure_registry_read_permission<'a>( &self, permissions: &mut dyn NodePermissions, - path: &Path, - ) -> Result<(), AnyError> { + path: &'a Path, + ) -> Result, AnyError> { + if permissions.query_read_all() { + return Ok(Cow::Borrowed(path)); // skip permissions checks below + } + // allow reading if it's in the node_modules let is_path_in_node_modules = path.starts_with(&self.registry_path) && path @@ -118,20 +120,20 @@ impl RegistryReadPermissionChecker { }, } }; - let Some(registry_path_canon) = canonicalize(&self.registry_path)? else { - return Ok(()); // not exists, allow reading - }; - let Some(path_canon) = canonicalize(path)? else { - return Ok(()); // not exists, allow reading - }; - - if path_canon.starts_with(registry_path_canon) { - return Ok(()); + if let Some(registry_path_canon) = canonicalize(&self.registry_path)? { + if let Some(path_canon) = canonicalize(path)? { + if path_canon.starts_with(registry_path_canon) { + return Ok(Cow::Owned(path_canon)); + } + } else if path.starts_with(registry_path_canon) + || path.starts_with(&self.registry_path) + { + return Ok(Cow::Borrowed(path)); + } } } - _ = permissions.check_read_path(path)?; - Ok(()) + permissions.check_read_path(path).map_err(Into::into) } } diff --git a/cli/npm/managed/resolvers/common/bin_entries.rs b/cli/npm/managed/resolvers/common/bin_entries.rs index 25a020c2bc614a..e4a1845689c155 100644 --- a/cli/npm/managed/resolvers/common/bin_entries.rs +++ b/cli/npm/managed/resolvers/common/bin_entries.rs @@ -18,6 +18,7 @@ pub struct BinEntries<'a> { seen_names: HashMap<&'a str, &'a NpmPackageId>, /// The bin entries entries: Vec<(&'a NpmResolutionPackage, PathBuf)>, + sorted: bool, } /// Returns the name of the default binary for the given package. @@ -31,6 +32,20 @@ fn default_bin_name(package: &NpmResolutionPackage) -> &str { .map_or(package.id.nv.name.as_str(), |(_, name)| name) } +pub fn warn_missing_entrypoint( + bin_name: &str, + package_path: &Path, + entrypoint: &Path, +) { + log::warn!( + "{} Trying to set up '{}' bin for \"{}\", but the entry point \"{}\" doesn't exist.", + deno_terminal::colors::yellow("Warning"), + bin_name, + package_path.display(), + entrypoint.display() + ); +} + impl<'a> BinEntries<'a> { pub fn new() -> Self { Self::default() @@ -42,6 +57,7 @@ impl<'a> BinEntries<'a> { package: &'a NpmResolutionPackage, package_path: PathBuf, ) { + self.sorted = false; // check for a new collision, if we haven't already // found one match package.bin.as_ref().unwrap() { @@ -69,39 +85,50 @@ impl<'a> BinEntries<'a> { fn for_each_entry( &mut self, snapshot: &NpmResolutionSnapshot, - mut f: impl FnMut( + mut already_seen: impl FnMut( + &Path, + &str, // bin script + ) -> Result<(), AnyError>, + mut new: impl FnMut( &NpmResolutionPackage, &Path, &str, // bin name &str, // bin script ) -> Result<(), AnyError>, + mut filter: impl FnMut(&NpmResolutionPackage) -> bool, ) -> Result<(), AnyError> { - if !self.collisions.is_empty() { + if !self.collisions.is_empty() && !self.sorted { // walking the dependency tree to find out the depth of each package // is sort of expensive, so we only do it if there's a collision sort_by_depth(snapshot, &mut self.entries, &mut self.collisions); + self.sorted = true; } let mut seen = HashSet::new(); for (package, package_path) in &self.entries { + if !filter(package) { + continue; + } if let Some(bin_entries) = &package.bin { match bin_entries { deno_npm::registry::NpmPackageVersionBinEntry::String(script) => { let name = default_bin_name(package); if !seen.insert(name) { + already_seen(package_path, script)?; // we already set up a bin entry with this name continue; } - f(package, package_path, name, script)?; + new(package, package_path, name, script)?; } deno_npm::registry::NpmPackageVersionBinEntry::Map(entries) => { for (name, script) in entries { if !seen.insert(name) { + already_seen(package_path, script)?; // we already set up a bin entry with this name continue; } - f(package, package_path, name, script)?; + new(package, package_path, name, script)?; } } } @@ -112,26 +139,31 @@ impl<'a> BinEntries<'a> { } /// Collect the bin entries into a vec of (name, script path) - pub fn into_bin_files( - mut self, + pub fn collect_bin_files( + &mut self, snapshot: &NpmResolutionSnapshot, ) -> Vec<(String, PathBuf)> { let mut bins = Vec::new(); self - .for_each_entry(snapshot, |_, package_path, name, script| { - bins.push((name.to_string(), package_path.join(script))); - Ok(()) - }) + .for_each_entry( + snapshot, + |_, _| Ok(()), + |_, package_path, name, script| { + bins.push((name.to_string(), package_path.join(script))); + Ok(()) + }, + |_| true, + ) .unwrap(); bins } - /// Finish setting up the bin entries, writing the necessary files - /// to disk. - pub fn finish( + fn set_up_entries_filtered( mut self, snapshot: &NpmResolutionSnapshot, bin_node_modules_dir_path: &Path, + filter: impl FnMut(&NpmResolutionPackage) -> bool, + mut handler: impl FnMut(&EntrySetupOutcome<'_>), ) -> Result<(), AnyError> { if !self.entries.is_empty() && !bin_node_modules_dir_path.exists() { std::fs::create_dir_all(bin_node_modules_dir_path).with_context( @@ -139,18 +171,65 @@ impl<'a> BinEntries<'a> { )?; } - self.for_each_entry(snapshot, |package, package_path, name, script| { - set_up_bin_entry( - package, - name, - script, - package_path, - bin_node_modules_dir_path, - ) - })?; + self.for_each_entry( + snapshot, + |_package_path, _script| { + #[cfg(unix)] + { + let path = _package_path.join(_script); + make_executable_if_exists(&path)?; + } + Ok(()) + }, + |package, package_path, name, script| { + let outcome = set_up_bin_entry( + package, + name, + script, + package_path, + bin_node_modules_dir_path, + )?; + handler(&outcome); + Ok(()) + }, + filter, + )?; Ok(()) } + + /// Finish setting up the bin entries, writing the necessary files + /// to disk. + pub fn finish( + self, + snapshot: &NpmResolutionSnapshot, + bin_node_modules_dir_path: &Path, + handler: impl FnMut(&EntrySetupOutcome<'_>), + ) -> Result<(), AnyError> { + self.set_up_entries_filtered( + snapshot, + bin_node_modules_dir_path, + |_| true, + handler, + ) + } + + /// Finish setting up the bin entries, writing the necessary files + /// to disk. + pub fn finish_only( + self, + snapshot: &NpmResolutionSnapshot, + bin_node_modules_dir_path: &Path, + handler: impl FnMut(&EntrySetupOutcome<'_>), + only: &HashSet<&NpmPackageId>, + ) -> Result<(), AnyError> { + self.set_up_entries_filtered( + snapshot, + bin_node_modules_dir_path, + |package| only.contains(&package.id), + handler, + ) + } } // walk the dependency tree to find out the depth of each package @@ -212,16 +291,17 @@ fn sort_by_depth( }); } -pub fn set_up_bin_entry( - package: &NpmResolutionPackage, - bin_name: &str, +pub fn set_up_bin_entry<'a>( + package: &'a NpmResolutionPackage, + bin_name: &'a str, #[allow(unused_variables)] bin_script: &str, - #[allow(unused_variables)] package_path: &Path, + #[allow(unused_variables)] package_path: &'a Path, bin_node_modules_dir_path: &Path, -) -> Result<(), AnyError> { +) -> Result, AnyError> { #[cfg(windows)] { set_up_bin_shim(package, bin_name, bin_node_modules_dir_path)?; + Ok(EntrySetupOutcome::Success) } #[cfg(unix)] { @@ -231,9 +311,8 @@ pub fn set_up_bin_entry( bin_script, package_path, bin_node_modules_dir_path, - )?; + ) } - Ok(()) } #[cfg(windows)] @@ -255,44 +334,81 @@ fn set_up_bin_shim( } #[cfg(unix)] -fn symlink_bin_entry( - _package: &NpmResolutionPackage, - bin_name: &str, - bin_script: &str, - package_path: &Path, - bin_node_modules_dir_path: &Path, -) -> Result<(), AnyError> { +/// Make the file at `path` executable if it exists. +/// Returns `true` if the file exists, `false` otherwise. +fn make_executable_if_exists(path: &Path) -> Result { use std::io; - use std::os::unix::fs::symlink; - let link = bin_node_modules_dir_path.join(bin_name); - let original = package_path.join(bin_script); - use std::os::unix::fs::PermissionsExt; - let mut perms = match std::fs::metadata(&original) { + let mut perms = match std::fs::metadata(path) { Ok(metadata) => metadata.permissions(), Err(err) => { if err.kind() == io::ErrorKind::NotFound { - log::warn!( - "{} Trying to set up '{}' bin for \"{}\", but the entry point \"{}\" doesn't exist.", - deno_terminal::colors::yellow("Warning"), - bin_name, - package_path.display(), - original.display() - ); - return Ok(()); + return Ok(false); } - return Err(err).with_context(|| { - format!("Can't set up '{}' bin at {}", bin_name, original.display()) - }); + return Err(err.into()); } }; if perms.mode() & 0o111 == 0 { // if the original file is not executable, make it executable perms.set_mode(perms.mode() | 0o111); - std::fs::set_permissions(&original, perms).with_context(|| { - format!("Setting permissions on '{}'", original.display()) + std::fs::set_permissions(path, perms).with_context(|| { + format!("Setting permissions on '{}'", path.display()) })?; } + + Ok(true) +} + +pub enum EntrySetupOutcome<'a> { + #[cfg_attr(windows, allow(dead_code))] + MissingEntrypoint { + bin_name: &'a str, + package_path: &'a Path, + entrypoint: PathBuf, + package: &'a NpmResolutionPackage, + }, + Success, +} + +impl<'a> EntrySetupOutcome<'a> { + pub fn warn_if_failed(&self) { + match self { + EntrySetupOutcome::MissingEntrypoint { + bin_name, + package_path, + entrypoint, + .. + } => warn_missing_entrypoint(bin_name, package_path, entrypoint), + EntrySetupOutcome::Success => {} + } + } +} + +#[cfg(unix)] +fn symlink_bin_entry<'a>( + package: &'a NpmResolutionPackage, + bin_name: &'a str, + bin_script: &str, + package_path: &'a Path, + bin_node_modules_dir_path: &Path, +) -> Result, AnyError> { + use std::io; + use std::os::unix::fs::symlink; + let link = bin_node_modules_dir_path.join(bin_name); + let original = package_path.join(bin_script); + + let found = make_executable_if_exists(&original).with_context(|| { + format!("Can't set up '{}' bin at {}", bin_name, original.display()) + })?; + if !found { + return Ok(EntrySetupOutcome::MissingEntrypoint { + bin_name, + package_path, + entrypoint: original, + package, + }); + } + let original_relative = crate::util::path::relative_path(bin_node_modules_dir_path, &original) .unwrap_or(original); @@ -313,7 +429,7 @@ fn symlink_bin_entry( original_relative.display() ) })?; - return Ok(()); + return Ok(EntrySetupOutcome::Success); } return Err(err).with_context(|| { format!( @@ -324,5 +440,5 @@ fn symlink_bin_entry( }); } - Ok(()) + Ok(EntrySetupOutcome::Success) } diff --git a/cli/npm/managed/resolvers/common/lifecycle_scripts.rs b/cli/npm/managed/resolvers/common/lifecycle_scripts.rs index a3c72634b3552e..f8b9e8a7e8573f 100644 --- a/cli/npm/managed/resolvers/common/lifecycle_scripts.rs +++ b/cli/npm/managed/resolvers/common/lifecycle_scripts.rs @@ -2,9 +2,15 @@ use super::bin_entries::BinEntries; use crate::args::LifecycleScriptsConfig; +use crate::task_runner::TaskStdio; +use crate::util::progress_bar::ProgressBar; +use deno_core::anyhow::Context; use deno_npm::resolution::NpmResolutionSnapshot; +use deno_runtime::deno_io::FromRawIoHandle; use deno_semver::package::PackageNv; +use deno_semver::Version; use std::borrow::Cow; +use std::collections::HashSet; use std::rc::Rc; use std::path::Path; @@ -56,7 +62,7 @@ impl<'a> LifecycleScripts<'a> { } } -fn has_lifecycle_scripts( +pub fn has_lifecycle_scripts( package: &NpmResolutionPackage, package_path: &Path, ) -> bool { @@ -78,7 +84,7 @@ fn is_broken_default_install_script(script: &str, package_path: &Path) -> bool { } impl<'a> LifecycleScripts<'a> { - fn can_run_scripts(&self, package_nv: &PackageNv) -> bool { + pub fn can_run_scripts(&self, package_nv: &PackageNv) -> bool { if !self.strategy.can_run_scripts() { return false; } @@ -93,6 +99,9 @@ impl<'a> LifecycleScripts<'a> { PackagesAllowedScripts::None => false, } } + pub fn has_run_scripts(&self, package: &NpmResolutionPackage) -> bool { + self.strategy.has_run(package) + } /// Register a package for running lifecycle scripts, if applicable. /// /// `package_path` is the path containing the package's code (its root dir). @@ -105,14 +114,25 @@ impl<'a> LifecycleScripts<'a> { ) { if has_lifecycle_scripts(package, &package_path) { if self.can_run_scripts(&package.id.nv) { - if !self.strategy.has_run(package) { + if !self.has_run_scripts(package) { self .packages_with_scripts .push((package, package_path.into_owned())); } - } else if !self.strategy.has_run(package) - && !self.strategy.has_warned(package) + } else if !self.has_run_scripts(package) + && (self.config.explicit_install || !self.strategy.has_warned(package)) { + // Skip adding `esbuild` as it is known that it can work properly without lifecycle script + // being run, and it's also very popular - any project using Vite would raise warnings. + { + let nv = &package.id.nv; + if nv.name == "esbuild" + && nv.version >= Version::parse_standard("0.18.0").unwrap() + { + return; + } + } + self .packages_with_scripts_not_run .push((package, package_path.into_owned())); @@ -133,27 +153,59 @@ impl<'a> LifecycleScripts<'a> { self, snapshot: &NpmResolutionSnapshot, packages: &[NpmResolutionPackage], - root_node_modules_dir_path: Option<&Path>, + root_node_modules_dir_path: &Path, + progress_bar: &ProgressBar, ) -> Result<(), AnyError> { self.warn_not_run_scripts()?; let get_package_path = |p: &NpmResolutionPackage| self.strategy.package_path(p); let mut failed_packages = Vec::new(); + let mut bin_entries = BinEntries::new(); if !self.packages_with_scripts.is_empty() { + let package_ids = self + .packages_with_scripts + .iter() + .map(|(p, _)| &p.id) + .collect::>(); // get custom commands for each bin available in the node_modules dir (essentially // the scripts that are in `node_modules/.bin`) - let base = - resolve_baseline_custom_commands(snapshot, packages, get_package_path)?; + let base = resolve_baseline_custom_commands( + &mut bin_entries, + snapshot, + packages, + get_package_path, + )?; let init_cwd = &self.config.initial_cwd; let process_state = crate::npm::managed::npm_process_state( snapshot.as_valid_serialized(), - root_node_modules_dir_path, + Some(root_node_modules_dir_path), ); let mut env_vars = crate::task_runner::real_env_vars(); + // so the subprocess can detect that it is running as part of a lifecycle script, + // and avoid trying to set up node_modules again env_vars.insert( - crate::args::NPM_RESOLUTION_STATE_ENV_VAR_NAME.to_string(), - process_state, + LIFECYCLE_SCRIPTS_RUNNING_ENV_VAR.to_string(), + "1".to_string(), + ); + // we want to pass the current state of npm resolution down to the deno subprocess + // (that may be running as part of the script). we do this with an inherited temp file + // + // SAFETY: we are sharing a single temp file across all of the scripts. the file position + // will be shared among these, which is okay since we run only one script at a time. + // However, if we concurrently run scripts in the future we will + // have to have multiple temp files. + let temp_file_fd = + deno_runtime::ops::process::npm_process_state_tempfile( + process_state.as_bytes(), + ).context("failed to create npm process state tempfile for running lifecycle scripts")?; + // SAFETY: fd/handle is valid + let _temp_file = + unsafe { std::fs::File::from_raw_io_handle(temp_file_fd) }; // make sure the file gets closed + env_vars.insert( + deno_runtime::ops::process::NPM_RESOLUTION_STATE_FD_ENV_VAR_NAME + .to_string(), + (temp_file_fd as usize).to_string(), ); for (package, package_path) in self.packages_with_scripts { // add custom commands for binaries from the package's dependencies. this will take precedence over the @@ -172,7 +224,15 @@ impl<'a> LifecycleScripts<'a> { { continue; } - let exit_code = crate::task_runner::run_task( + let _guard = progress_bar.update_with_prompt( + crate::util::progress_bar::ProgressMessagePrompt::Initialize, + &format!("{}: running '{script_name}' script", package.id.nv), + ); + let crate::task_runner::TaskResult { + exit_code, + stderr, + stdout, + } = crate::task_runner::run_task( crate::task_runner::RunTaskOptions { task_name: script_name, script, @@ -181,16 +241,38 @@ impl<'a> LifecycleScripts<'a> { custom_commands: custom_commands.clone(), init_cwd, argv: &[], - root_node_modules_dir: root_node_modules_dir_path, + root_node_modules_dir: Some(root_node_modules_dir_path), + stdio: Some(crate::task_runner::TaskIo { + stderr: TaskStdio::piped(), + stdout: TaskStdio::piped(), + }), }, ) .await?; + let stdout = stdout.unwrap(); + let stderr = stderr.unwrap(); if exit_code != 0 { log::warn!( - "error: script '{}' in '{}' failed with exit code {}", + "error: script '{}' in '{}' failed with exit code {}{}{}", script_name, package.id.nv, exit_code, + if !stdout.trim_ascii().is_empty() { + format!( + "\nstdout:\n{}\n", + String::from_utf8_lossy(&stdout).trim() + ) + } else { + String::new() + }, + if !stderr.trim_ascii().is_empty() { + format!( + "\nstderr:\n{}\n", + String::from_utf8_lossy(&stderr).trim() + ) + } else { + String::new() + }, ); failed_packages.push(&package.id.nv); // assume if earlier script fails, later ones will fail too @@ -200,6 +282,17 @@ impl<'a> LifecycleScripts<'a> { } self.strategy.did_run_scripts(package)?; } + + // re-set up bin entries for the packages which we've run scripts for. + // lifecycle scripts can create files that are linked to by bin entries, + // and the only reliable way to handle this is to re-link bin entries + // (this is what PNPM does as well) + bin_entries.finish_only( + snapshot, + &root_node_modules_dir_path.join(".bin"), + |outcome| outcome.warn_if_failed(), + &package_ids, + )?; } if failed_packages.is_empty() { Ok(()) @@ -216,12 +309,20 @@ impl<'a> LifecycleScripts<'a> { } } +const LIFECYCLE_SCRIPTS_RUNNING_ENV_VAR: &str = + "DENO_INTERNAL_IS_LIFECYCLE_SCRIPT"; + +pub fn is_running_lifecycle_script() -> bool { + std::env::var(LIFECYCLE_SCRIPTS_RUNNING_ENV_VAR).is_ok() +} + // take in all (non copy) packages from snapshot, // and resolve the set of available binaries to create // custom commands available to the task runner -fn resolve_baseline_custom_commands( - snapshot: &NpmResolutionSnapshot, - packages: &[NpmResolutionPackage], +fn resolve_baseline_custom_commands<'a>( + bin_entries: &mut BinEntries<'a>, + snapshot: &'a NpmResolutionSnapshot, + packages: &'a [NpmResolutionPackage], get_package_path: impl Fn(&NpmResolutionPackage) -> PathBuf, ) -> Result { let mut custom_commands = crate::task_runner::TaskCustomCommands::new(); @@ -244,6 +345,7 @@ fn resolve_baseline_custom_commands( // doing it for packages that are set up already. // realistically, scripts won't be run very often so it probably isn't too big of an issue. resolve_custom_commands_from_packages( + bin_entries, custom_commands, snapshot, packages, @@ -258,12 +360,12 @@ fn resolve_custom_commands_from_packages< 'a, P: IntoIterator, >( + bin_entries: &mut BinEntries<'a>, mut commands: crate::task_runner::TaskCustomCommands, snapshot: &'a NpmResolutionSnapshot, packages: P, get_package_path: impl Fn(&'a NpmResolutionPackage) -> PathBuf, ) -> Result { - let mut bin_entries = BinEntries::new(); for package in packages { let package_path = get_package_path(package); @@ -271,7 +373,7 @@ fn resolve_custom_commands_from_packages< bin_entries.add(package, package_path); } } - let bins = bin_entries.into_bin_files(snapshot); + let bins: Vec<(String, PathBuf)> = bin_entries.collect_bin_files(snapshot); for (bin_name, script_path) in bins { commands.insert( bin_name.clone(), @@ -294,7 +396,9 @@ fn resolve_custom_commands_from_deps( snapshot: &NpmResolutionSnapshot, get_package_path: impl Fn(&NpmResolutionPackage) -> PathBuf, ) -> Result { + let mut bin_entries = BinEntries::new(); resolve_custom_commands_from_packages( + &mut bin_entries, baseline, snapshot, package diff --git a/cli/npm/managed/resolvers/global.rs b/cli/npm/managed/resolvers/global.rs index 187e6b2774c5d1..f0193e78e95cf2 100644 --- a/cli/npm/managed/resolvers/global.rs +++ b/cli/npm/managed/resolvers/global.rs @@ -11,7 +11,6 @@ use crate::colors; use async_trait::async_trait; use deno_ast::ModuleSpecifier; use deno_core::error::AnyError; -use deno_core::url::Url; use deno_npm::NpmPackageCacheFolderId; use deno_npm::NpmPackageId; use deno_npm::NpmResolutionPackage; @@ -56,7 +55,7 @@ impl GlobalNpmPackageResolver { Self { registry_read_permission_checker: RegistryReadPermissionChecker::new( fs, - cache.root_folder(), + cache.root_dir_path().to_path_buf(), ), cache, tarball_cache, @@ -69,11 +68,7 @@ impl GlobalNpmPackageResolver { #[async_trait(?Send)] impl NpmPackageFsResolver for GlobalNpmPackageResolver { - fn root_dir_url(&self) -> &Url { - self.cache.root_dir_url() - } - - fn node_modules_path(&self) -> Option<&PathBuf> { + fn node_modules_path(&self) -> Option<&Path> { None } @@ -183,11 +178,11 @@ impl NpmPackageFsResolver for GlobalNpmPackageResolver { Ok(()) } - fn ensure_read_permission( + fn ensure_read_permission<'a>( &self, permissions: &mut dyn NodePermissions, - path: &Path, - ) -> Result<(), AnyError> { + path: &'a Path, + ) -> Result, AnyError> { self .registry_read_permission_checker .ensure_registry_read_permission(permissions, path) diff --git a/cli/npm/managed/resolvers/local.rs b/cli/npm/managed/resolvers/local.rs index 5a90f252dec2d7..ca7867425d6419 100644 --- a/cli/npm/managed/resolvers/local.rs +++ b/cli/npm/managed/resolvers/local.rs @@ -19,6 +19,7 @@ use crate::args::LifecycleScriptsConfig; use crate::colors; use async_trait::async_trait; use deno_ast::ModuleSpecifier; +use deno_cache_dir::npm::mixed_case_package_name_decode; use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::futures::stream::FuturesUnordered; @@ -30,6 +31,7 @@ use deno_npm::NpmPackageCacheFolderId; use deno_npm::NpmPackageId; use deno_npm::NpmResolutionPackage; use deno_npm::NpmSystemInfo; +use deno_resolver::npm::normalize_pkg_name_for_node_modules_deno_folder; use deno_runtime::deno_fs; use deno_runtime::deno_node::NodePermissions; use deno_semver::package::PackageNv; @@ -42,8 +44,6 @@ use serde::Serialize; use crate::args::NpmInstallDepsProvider; use crate::cache::CACHE_PERM; -use crate::npm::cache_dir::mixed_case_package_name_decode; -use crate::npm::cache_dir::mixed_case_package_name_encode; use crate::util::fs::atomic_write_file_with_retries; use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; use crate::util::fs::clone_dir_recursive; @@ -55,6 +55,7 @@ use crate::util::progress_bar::ProgressMessagePrompt; use super::super::cache::NpmCache; use super::super::cache::TarballCache; use super::super::resolution::NpmResolution; +use super::common::bin_entries; use super::common::NpmPackageFsResolver; use super::common::RegistryReadPermissionChecker; @@ -155,12 +156,8 @@ impl LocalNpmPackageResolver { #[async_trait(?Send)] impl NpmPackageFsResolver for LocalNpmPackageResolver { - fn root_dir_url(&self) -> &Url { - &self.root_node_modules_url - } - - fn node_modules_path(&self) -> Option<&PathBuf> { - Some(&self.root_node_modules_path) + fn node_modules_path(&self) -> Option<&Path> { + Some(self.root_node_modules_path.as_ref()) } fn maybe_package_folder(&self, id: &NpmPackageId) -> Option { @@ -257,11 +254,11 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver { .await } - fn ensure_read_permission( + fn ensure_read_permission<'a>( &self, permissions: &mut dyn NodePermissions, - path: &Path, - ) -> Result<(), AnyError> { + path: &'a Path, + ) -> Result, AnyError> { self .registry_read_permission_checker .ensure_registry_read_permission(permissions, path) @@ -301,6 +298,12 @@ async fn sync_resolution_with_fs( return Ok(()); // don't create the directory } + // don't set up node_modules (and more importantly try to acquire the file lock) + // if we're running as part of a lifecycle script + if super::common::lifecycle_scripts::is_running_lifecycle_script() { + return Ok(()); + } + let deno_local_registry_dir = root_node_modules_dir_path.join(".deno"); let deno_node_modules_dir = deno_local_registry_dir.join("node_modules"); fs::create_dir_all(&deno_node_modules_dir).with_context(|| { @@ -333,8 +336,7 @@ async fn sync_resolution_with_fs( let mut cache_futures = FuturesUnordered::new(); let mut newest_packages_by_name: HashMap<&String, &NpmResolutionPackage> = HashMap::with_capacity(package_partitions.packages.len()); - let bin_entries = - Rc::new(RefCell::new(super::common::bin_entries::BinEntries::new())); + let bin_entries = Rc::new(RefCell::new(bin_entries::BinEntries::new())); let mut lifecycle_scripts = super::common::lifecycle_scripts::LifecycleScripts::new( lifecycle_scripts, @@ -343,6 +345,14 @@ async fn sync_resolution_with_fs( }, ); let packages_with_deprecation_warnings = Arc::new(Mutex::new(Vec::new())); + + let mut package_tags: HashMap<&PackageNv, Vec<&str>> = HashMap::new(); + for (package_req, package_nv) in snapshot.package_reqs() { + if let Some(tag) = package_req.version_req.tag() { + package_tags.entry(package_nv).or_default().push(tag); + } + } + for package in &package_partitions.packages { if let Some(current_pkg) = newest_packages_by_name.get_mut(&package.id.nv.name) @@ -357,11 +367,29 @@ async fn sync_resolution_with_fs( let package_folder_name = get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); let folder_path = deno_local_registry_dir.join(&package_folder_name); + let tags = package_tags + .get(&package.id.nv) + .map(|tags| tags.join(",")) + .unwrap_or_default(); + enum PackageFolderState { + UpToDate, + Uninitialized, + TagsOutdated, + } let initialized_file = folder_path.join(".initialized"); + let package_state = std::fs::read_to_string(&initialized_file) + .map(|s| { + if s != tags { + PackageFolderState::TagsOutdated + } else { + PackageFolderState::UpToDate + } + }) + .unwrap_or(PackageFolderState::Uninitialized); if !cache .cache_setting() .should_use_for_npm_package(&package.id.nv.name) - || !initialized_file.exists() + || matches!(package_state, PackageFolderState::Uninitialized) { // cache bust the dep from the dep setup cache so the symlinks // are forced to be recreated @@ -371,6 +399,7 @@ async fn sync_resolution_with_fs( let bin_entries_to_setup = bin_entries.clone(); let packages_with_deprecation_warnings = packages_with_deprecation_warnings.clone(); + cache_futures.push(async move { tarball_cache .ensure_package(&package.id.nv, &package.dist) @@ -389,7 +418,7 @@ async fn sync_resolution_with_fs( move || { clone_dir_recursive(&cache_folder, &package_path)?; // write out a file that indicates this folder has been initialized - fs::write(initialized_file, "")?; + fs::write(initialized_file, tags)?; Ok::<_, AnyError>(()) } @@ -410,6 +439,8 @@ async fn sync_resolution_with_fs( drop(pb_guard); // explicit for clarity Ok::<_, AnyError>(()) }); + } else if matches!(package_state, PackageFolderState::TagsOutdated) { + fs::write(initialized_file, tags)?; } let sub_node_modules = folder_path.join("node_modules"); @@ -518,9 +549,9 @@ async fn sync_resolution_with_fs( // linked into the root match found_names.entry(remote_alias) { Entry::Occupied(nv) => { - alias_clashes - || remote.req.name != nv.get().name // alias to a different package (in case of duplicate aliases) - || !remote.req.version_req.matches(&nv.get().version) // incompatible version + // alias to a different package (in case of duplicate aliases) + // or the version doesn't match the version in the root node_modules + alias_clashes || &remote_pkg.id.nv != *nv.get() } Entry::Vacant(entry) => { entry.insert(&remote_pkg.id.nv); @@ -633,7 +664,28 @@ async fn sync_resolution_with_fs( // 7. Set up `node_modules/.bin` entries for packages that need it. { let bin_entries = std::mem::take(&mut *bin_entries.borrow_mut()); - bin_entries.finish(snapshot, &bin_node_modules_dir_path)?; + bin_entries.finish( + snapshot, + &bin_node_modules_dir_path, + |setup_outcome| { + match setup_outcome { + bin_entries::EntrySetupOutcome::MissingEntrypoint { + package, + package_path, + .. + } if super::common::lifecycle_scripts::has_lifecycle_scripts( + package, + package_path, + ) && lifecycle_scripts.can_run_scripts(&package.id.nv) + && !lifecycle_scripts.has_run_scripts(package) => + { + // ignore, it might get fixed when the lifecycle scripts run. + // if not, we'll warn then + } + outcome => outcome.warn_if_failed(), + } + }, + )?; } // 8. Create symlinks for the workspace packages @@ -683,7 +735,8 @@ async fn sync_resolution_with_fs( .finish( snapshot, &package_partitions.packages, - Some(root_node_modules_dir_path), + root_node_modules_dir_path, + progress_bar, ) .await?; @@ -920,20 +973,6 @@ impl SetupCache { } } -/// Normalizes a package name for use at `node_modules/.deno/@[_]` -pub fn normalize_pkg_name_for_node_modules_deno_folder(name: &str) -> Cow { - let name = if name.to_lowercase() == name { - Cow::Borrowed(name) - } else { - Cow::Owned(format!("_{}", mixed_case_package_name_encode(name))) - }; - if name.starts_with('@') { - name.replace('/', "+").into() - } else { - name - } -} - fn get_package_folder_id_folder_name( folder_id: &NpmPackageCacheFolderId, ) -> String { @@ -1023,12 +1062,18 @@ fn junction_or_symlink_dir( if symlink_err.kind() == std::io::ErrorKind::PermissionDenied => { USE_JUNCTIONS.store(true, std::sync::atomic::Ordering::Relaxed); - junction::create(old_path, new_path).map_err(Into::into) + junction::create(old_path, new_path) + .context("Failed creating junction in node_modules folder") + } + Err(symlink_err) => { + log::warn!( + "{} Unexpected error symlinking node_modules: {symlink_err}", + colors::yellow("Warning") + ); + USE_JUNCTIONS.store(true, std::sync::atomic::Ordering::Relaxed); + junction::create(old_path, new_path) + .context("Failed creating junction in node_modules folder") } - Err(symlink_err) => Err( - AnyError::from(symlink_err) - .context("Failed creating symlink in node_modules folder"), - ), } } diff --git a/cli/npm/managed/resolvers/mod.rs b/cli/npm/managed/resolvers/mod.rs index 234a6e4dba99ac..36d795ee7e7bc6 100644 --- a/cli/npm/managed/resolvers/mod.rs +++ b/cli/npm/managed/resolvers/mod.rs @@ -15,7 +15,6 @@ use crate::args::NpmInstallDepsProvider; use crate::util::progress_bar::ProgressBar; pub use self::common::NpmPackageFsResolver; -pub use self::local::normalize_pkg_name_for_node_modules_deno_folder; use self::global::GlobalNpmPackageResolver; use self::local::LocalNpmPackageResolver; diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index bedde6455175f2..0e955ac5b43396 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -1,37 +1,44 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. mod byonm; -mod cache_dir; mod common; mod managed; -use std::path::PathBuf; +use std::borrow::Cow; +use std::path::Path; use std::sync::Arc; +use common::maybe_auth_header_for_npm_registry; use dashmap::DashMap; -use deno_ast::ModuleSpecifier; use deno_core::error::AnyError; use deno_core::serde_json; +use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::registry::NpmPackageInfo; -use deno_runtime::deno_node::NodeRequireResolver; -use deno_runtime::deno_node::NpmProcessStateProvider; +use deno_resolver::npm::ByonmInNpmPackageChecker; +use deno_resolver::npm::ByonmNpmResolver; +use deno_resolver::npm::CliNpmReqResolver; +use deno_resolver::npm::ResolvePkgFolderFromDenoReqError; +use deno_runtime::deno_node::NodePermissions; +use deno_runtime::ops::process::NpmProcessStateProvider; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; -use node_resolver::NpmResolver; +use managed::cache::registry_info::get_package_url; +use managed::create_managed_in_npm_pkg_checker; +use node_resolver::InNpmPackageChecker; +use node_resolver::NpmPackageFolderResolver; -use crate::args::npm_registry_url; use crate::file_fetcher::FileFetcher; -pub use self::byonm::ByonmCliNpmResolver; -pub use self::byonm::CliNpmResolverByonmCreateOptions; -pub use self::cache_dir::NpmCacheDir; -pub use self::managed::CliNpmResolverManagedCreateOptions; +pub use self::byonm::CliByonmNpmResolver; +pub use self::byonm::CliByonmNpmResolverCreateOptions; +pub use self::managed::CliManagedInNpmPkgCheckerCreateOptions; +pub use self::managed::CliManagedNpmResolverCreateOptions; pub use self::managed::CliNpmResolverManagedSnapshotOption; pub use self::managed::ManagedCliNpmResolver; pub enum CliNpmResolverCreateOptions { - Managed(CliNpmResolverManagedCreateOptions), - Byonm(CliNpmResolverByonmCreateOptions), + Managed(CliManagedNpmResolverCreateOptions), + Byonm(CliByonmNpmResolverCreateOptions), } pub async fn create_cli_npm_resolver_for_lsp( @@ -42,7 +49,7 @@ pub async fn create_cli_npm_resolver_for_lsp( Managed(options) => { managed::create_managed_npm_resolver_for_lsp(options).await } - Byonm(options) => byonm::create_byonm_npm_resolver(options), + Byonm(options) => Arc::new(ByonmNpmResolver::new(options)), } } @@ -52,22 +59,43 @@ pub async fn create_cli_npm_resolver( use CliNpmResolverCreateOptions::*; match options { Managed(options) => managed::create_managed_npm_resolver(options).await, - Byonm(options) => Ok(byonm::create_byonm_npm_resolver(options)), + Byonm(options) => Ok(Arc::new(ByonmNpmResolver::new(options))), + } +} + +pub enum CreateInNpmPkgCheckerOptions<'a> { + Managed(CliManagedInNpmPkgCheckerCreateOptions<'a>), + Byonm, +} + +pub fn create_in_npm_pkg_checker( + options: CreateInNpmPkgCheckerOptions, +) -> Arc { + match options { + CreateInNpmPkgCheckerOptions::Managed(options) => { + create_managed_in_npm_pkg_checker(options) + } + CreateInNpmPkgCheckerOptions::Byonm => Arc::new(ByonmInNpmPackageChecker), } } pub enum InnerCliNpmResolverRef<'a> { Managed(&'a ManagedCliNpmResolver), #[allow(dead_code)] - Byonm(&'a ByonmCliNpmResolver), + Byonm(&'a CliByonmNpmResolver), } -pub trait CliNpmResolver: NpmResolver { - fn into_npm_resolver(self: Arc) -> Arc; - fn into_require_resolver(self: Arc) -> Arc; +pub trait CliNpmResolver: NpmPackageFolderResolver + CliNpmReqResolver { + fn into_npm_pkg_folder_resolver( + self: Arc, + ) -> Arc; + fn into_npm_req_resolver(self: Arc) -> Arc; fn into_process_state_provider( self: Arc, ) -> Arc; + fn into_maybe_byonm(self: Arc) -> Option> { + None + } fn clone_snapshotted(&self) -> Arc; @@ -80,20 +108,20 @@ pub trait CliNpmResolver: NpmResolver { } } - fn as_byonm(&self) -> Option<&ByonmCliNpmResolver> { + fn as_byonm(&self) -> Option<&CliByonmNpmResolver> { match self.as_inner() { InnerCliNpmResolverRef::Managed(_) => None, InnerCliNpmResolverRef::Byonm(inner) => Some(inner), } } - fn root_node_modules_path(&self) -> Option<&PathBuf>; + fn root_node_modules_path(&self) -> Option<&Path>; - fn resolve_pkg_folder_from_deno_module_req( + fn ensure_read_permission<'a>( &self, - req: &PackageReq, - referrer: &ModuleSpecifier, - ) -> Result; + permissions: &mut dyn NodePermissions, + path: &'a Path, + ) -> Result, AnyError>; /// Returns a hash returning the state of the npm resolver /// or `None` if the state currently can't be determined. @@ -105,14 +133,19 @@ pub struct NpmFetchResolver { nv_by_req: DashMap>, info_by_name: DashMap>>, file_fetcher: Arc, + npmrc: Arc, } impl NpmFetchResolver { - pub fn new(file_fetcher: Arc) -> Self { + pub fn new( + file_fetcher: Arc, + npmrc: Arc, + ) -> Self { Self { nv_by_req: Default::default(), info_by_name: Default::default(), file_fetcher, + npmrc, } } @@ -147,11 +180,21 @@ impl NpmFetchResolver { return info.value().clone(); } let fetch_package_info = || async { - let info_url = npm_registry_url().join(name).ok()?; + let info_url = get_package_url(&self.npmrc, name); let file_fetcher = self.file_fetcher.clone(); + let registry_config = self.npmrc.get_registry_config(name); + // TODO(bartlomieju): this should error out, not use `.ok()`. + let maybe_auth_header = + maybe_auth_header_for_npm_registry(registry_config).ok()?; // spawn due to the lsp's `Send` requirement let file = deno_core::unsync::spawn(async move { - file_fetcher.fetch_bypass_permissions(&info_url).await.ok() + file_fetcher + .fetch_bypass_permissions_with_maybe_auth( + &info_url, + maybe_auth_header, + ) + .await + .ok() }) .await .ok()??; @@ -162,3 +205,15 @@ impl NpmFetchResolver { info } } + +pub const NPM_CONFIG_USER_AGENT_ENV_VAR: &str = "npm_config_user_agent"; + +pub fn get_npm_config_user_agent() -> String { + format!( + "deno/{} npm/? deno/{} {} {}", + env!("CARGO_PKG_VERSION"), + env!("CARGO_PKG_VERSION"), + std::env::consts::OS, + std::env::consts::ARCH + ) +} diff --git a/cli/ops/bench.rs b/cli/ops/bench.rs index edd8c118caff51..a7c788a1899884 100644 --- a/cli/ops/bench.rs +++ b/cli/ops/bench.rs @@ -2,8 +2,6 @@ use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; -use std::time; use deno_core::error::generic_error; use deno_core::error::type_error; @@ -12,10 +10,9 @@ use deno_core::op2; use deno_core::v8; use deno_core::ModuleSpecifier; use deno_core::OpState; -use deno_runtime::deno_permissions::create_child_permissions; use deno_runtime::deno_permissions::ChildPermissionsArg; -use deno_runtime::deno_permissions::PermissionDescriptorParser; use deno_runtime::deno_permissions::PermissionsContainer; +use deno_runtime::deno_web::StartTime; use tokio::sync::mpsc::UnboundedSender; use uuid::Uuid; @@ -54,26 +51,15 @@ fn op_bench_get_origin(state: &mut OpState) -> String { #[derive(Clone)] struct PermissionsHolder(Uuid, PermissionsContainer); -#[op2] +#[op2(stack_trace)] #[serde] pub fn op_pledge_test_permissions( state: &mut OpState, #[serde] args: ChildPermissionsArg, -) -> Result { +) -> Result { let token = Uuid::new_v4(); - let permission_desc_parser = state - .borrow::>() - .clone(); let parent_permissions = state.borrow_mut::(); - let worker_permissions = { - let mut parent_permissions = parent_permissions.inner.lock(); - let perms = create_child_permissions( - permission_desc_parser.as_ref(), - &mut parent_permissions, - args, - )?; - PermissionsContainer::new(permission_desc_parser, perms) - }; + let worker_permissions = parent_permissions.create_child_permissions(args)?; let parent_permissions = parent_permissions.clone(); if state.try_take::().is_some() { @@ -83,7 +69,6 @@ pub fn op_pledge_test_permissions( state.put::(PermissionsHolder(token, parent_permissions)); // NOTE: This call overrides current permission set for the worker - state.put(worker_permissions.inner.clone()); state.put::(worker_permissions); Ok(token) @@ -100,7 +85,6 @@ pub fn op_restore_test_permissions( } let permissions = permissions_holder.1; - state.put(permissions.inner.clone()); state.put::(permissions); Ok(()) } else { @@ -163,8 +147,8 @@ fn op_dispatch_bench_event(state: &mut OpState, #[serde] event: BenchEvent) { #[op2(fast)] #[number] -fn op_bench_now(state: &mut OpState) -> Result { - let ns = state.borrow::().elapsed().as_nanos(); +fn op_bench_now(state: &mut OpState) -> Result { + let ns = state.borrow::().elapsed().as_nanos(); let ns_u64 = u64::try_from(ns)?; Ok(ns_u64) } diff --git a/cli/ops/jupyter.rs b/cli/ops/jupyter.rs index f7f006d9bd506c..5bdf97e60f6c20 100644 --- a/cli/ops/jupyter.rs +++ b/cli/ops/jupyter.rs @@ -46,7 +46,7 @@ pub fn op_jupyter_input( state: &mut OpState, #[string] prompt: String, is_password: bool, -) -> Result, AnyError> { +) -> Option { let (last_execution_request, stdin_connection_proxy) = { ( state.borrow::>>>().clone(), @@ -58,11 +58,11 @@ pub fn op_jupyter_input( if let Some(last_request) = maybe_last_request { let JupyterMessageContent::ExecuteRequest(msg) = &last_request.content else { - return Ok(None); + return None; }; if !msg.allow_stdin { - return Ok(None); + return None; } let content = InputRequest { @@ -73,7 +73,7 @@ pub fn op_jupyter_input( let msg = JupyterMessage::new(content, Some(&last_request)); let Ok(()) = stdin_connection_proxy.lock().tx.send(msg) else { - return Ok(None); + return None; }; // Need to spawn a separate thread here, because `blocking_recv()` can't @@ -82,17 +82,25 @@ pub fn op_jupyter_input( stdin_connection_proxy.lock().rx.blocking_recv() }); let Ok(Some(response)) = join_handle.join() else { - return Ok(None); + return None; }; let JupyterMessageContent::InputReply(msg) = response.content else { - return Ok(None); + return None; }; - return Ok(Some(msg.value)); + return Some(msg.value); } - Ok(None) + None +} + +#[derive(Debug, thiserror::Error)] +pub enum JupyterBroadcastError { + #[error(transparent)] + SerdeJson(serde_json::Error), + #[error(transparent)] + ZeroMq(AnyError), } #[op2(async)] @@ -102,7 +110,7 @@ pub async fn op_jupyter_broadcast( #[serde] content: serde_json::Value, #[serde] metadata: serde_json::Value, #[serde] buffers: Vec, -) -> Result<(), AnyError> { +) -> Result<(), JupyterBroadcastError> { let (iopub_connection, last_execution_request) = { let s = state.borrow(); @@ -125,36 +133,35 @@ pub async fn op_jupyter_broadcast( content, err ); - err + JupyterBroadcastError::SerdeJson(err) })?; let jupyter_message = JupyterMessage::new(content, Some(&last_request)) .with_metadata(metadata) .with_buffers(buffers.into_iter().map(|b| b.to_vec().into()).collect()); - iopub_connection.lock().send(jupyter_message).await?; + iopub_connection + .lock() + .send(jupyter_message) + .await + .map_err(JupyterBroadcastError::ZeroMq)?; } Ok(()) } #[op2(fast)] -pub fn op_print( - state: &mut OpState, - #[string] msg: &str, - is_err: bool, -) -> Result<(), AnyError> { +pub fn op_print(state: &mut OpState, #[string] msg: &str, is_err: bool) { let sender = state.borrow_mut::>(); if is_err { if let Err(err) = sender.send(StreamContent::stderr(msg)) { log::error!("Failed to send stderr message: {}", err); } - return Ok(()); + return; } if let Err(err) = sender.send(StreamContent::stdout(msg)) { log::error!("Failed to send stdout message: {}", err); } - Ok(()) } diff --git a/cli/ops/testing.rs b/cli/ops/testing.rs index 6a8d31006298b8..3c6936971a5ae3 100644 --- a/cli/ops/testing.rs +++ b/cli/ops/testing.rs @@ -16,13 +16,10 @@ use deno_core::op2; use deno_core::v8; use deno_core::ModuleSpecifier; use deno_core::OpState; -use deno_runtime::deno_permissions::create_child_permissions; use deno_runtime::deno_permissions::ChildPermissionsArg; -use deno_runtime::deno_permissions::PermissionDescriptorParser; use deno_runtime::deno_permissions::PermissionsContainer; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use std::sync::Arc; use uuid::Uuid; deno_core::extension!(deno_test, @@ -49,26 +46,15 @@ deno_core::extension!(deno_test, #[derive(Clone)] struct PermissionsHolder(Uuid, PermissionsContainer); -#[op2] +#[op2(stack_trace)] #[serde] pub fn op_pledge_test_permissions( state: &mut OpState, #[serde] args: ChildPermissionsArg, -) -> Result { +) -> Result { let token = Uuid::new_v4(); - let permission_desc_parser = state - .borrow::>() - .clone(); let parent_permissions = state.borrow_mut::(); - let worker_permissions = { - let mut parent_permissions = parent_permissions.inner.lock(); - let perms = create_child_permissions( - permission_desc_parser.as_ref(), - &mut parent_permissions, - args, - )?; - PermissionsContainer::new(permission_desc_parser, perms) - }; + let worker_permissions = parent_permissions.create_child_permissions(args)?; let parent_permissions = parent_permissions.clone(); if state.try_take::().is_some() { @@ -77,7 +63,6 @@ pub fn op_pledge_test_permissions( state.put::(PermissionsHolder(token, parent_permissions)); // NOTE: This call overrides current permission set for the worker - state.put(worker_permissions.inner.clone()); state.put::(worker_permissions); Ok(token) @@ -94,7 +79,6 @@ pub fn op_restore_test_permissions( } let permissions = permissions_holder.1; - state.put(permissions.inner.clone()); state.put::(permissions); Ok(()) } else { @@ -166,7 +150,7 @@ fn op_register_test_step( #[smi] parent_id: usize, #[smi] root_id: usize, #[string] root_name: String, -) -> Result { +) -> usize { let id = NEXT_ID.fetch_add(1, Ordering::SeqCst); let origin = state.borrow::().to_string(); let description = TestStepDescription { @@ -185,7 +169,7 @@ fn op_register_test_step( }; let sender = state.borrow_mut::(); sender.send(TestEvent::StepRegister(description)).ok(); - Ok(id) + id } #[op2(fast)] diff --git a/cli/resolver.rs b/cli/resolver.rs index 07cb6931df5960..15ca4aa2b6ca21 100644 --- a/cli/resolver.rs +++ b/cli/resolver.rs @@ -4,55 +4,53 @@ use async_trait::async_trait; use dashmap::DashMap; use dashmap::DashSet; use deno_ast::MediaType; -use deno_config::workspace::MappedResolution; use deno_config::workspace::MappedResolutionDiagnostic; use deno_config::workspace::MappedResolutionError; -use deno_config::workspace::WorkspaceResolver; use deno_core::anyhow::anyhow; use deno_core::anyhow::Context; use deno_core::error::AnyError; +use deno_core::url::Url; use deno_core::ModuleSourceCode; use deno_core::ModuleSpecifier; -use deno_graph::source::ResolutionMode; use deno_graph::source::ResolveError; -use deno_graph::source::Resolver; use deno_graph::source::UnknownBuiltInNodeModuleError; -use deno_graph::source::DEFAULT_JSX_IMPORT_SOURCE_MODULE; use deno_graph::NpmLoadError; use deno_graph::NpmResolvePkgReqsResult; use deno_npm::resolution::NpmResolutionError; -use deno_package_json::PackageJsonDepValue; +use deno_resolver::sloppy_imports::SloppyImportsResolver; use deno_runtime::colors; use deno_runtime::deno_fs; use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_node::is_builtin_node_module; -use deno_runtime::deno_node::NodeResolver; -use deno_runtime::fs_util::specifier_to_file_path; -use deno_semver::npm::NpmPackageReqReference; +use deno_runtime::deno_node::DenoFsNodeResolverEnv; use deno_semver::package::PackageReq; -use node_resolver::errors::ClosestPkgJsonError; -use node_resolver::errors::NodeResolveError; -use node_resolver::errors::NodeResolveErrorKind; -use node_resolver::errors::PackageFolderResolveErrorKind; -use node_resolver::errors::PackageFolderResolveIoError; -use node_resolver::errors::PackageNotFoundError; -use node_resolver::errors::PackageResolveErrorKind; -use node_resolver::errors::UrlToNodeResolutionError; -use node_resolver::NodeModuleKind; -use node_resolver::NodeResolution; -use node_resolver::NodeResolutionMode; -use node_resolver::PackageJson; +use node_resolver::NodeResolutionKind; +use node_resolver::ResolutionMode; use std::borrow::Cow; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; +use thiserror::Error; -use crate::args::JsxImportSourceConfig; use crate::args::DENO_DISABLE_PEDANTIC_NODE_WARNINGS; use crate::node::CliNodeCodeTranslator; use crate::npm::CliNpmResolver; use crate::npm::InnerCliNpmResolverRef; use crate::util::sync::AtomicFlag; +use crate::util::text_encoding::from_utf8_lossy_cow; + +pub type CjsTracker = deno_resolver::cjs::CjsTracker; +pub type IsCjsResolver = + deno_resolver::cjs::IsCjsResolver; +pub type CliSloppyImportsResolver = + SloppyImportsResolver; +pub type CliDenoResolver = deno_resolver::DenoResolver< + CliDenoResolverFs, + DenoFsNodeResolverEnv, + SloppyImportsCachedFs, +>; +pub type CliNpmReqResolver = + deno_resolver::npm::NpmReqResolver; pub struct ModuleCodeStringSource { pub code: ModuleSourceCode, @@ -60,279 +58,78 @@ pub struct ModuleCodeStringSource { pub media_type: MediaType, } -#[derive(Debug)] -pub struct CliNodeResolver { - cjs_resolutions: Arc, - fs: Arc, - node_resolver: Arc, - // todo(dsherret): remove this pub(crate) - pub(crate) npm_resolver: Arc, -} - -impl CliNodeResolver { - pub fn new( - cjs_resolutions: Arc, - fs: Arc, - node_resolver: Arc, - npm_resolver: Arc, - ) -> Self { - Self { - cjs_resolutions, - fs, - node_resolver, - npm_resolver, - } - } - - pub fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool { - self.npm_resolver.in_npm_package(specifier) - } - - pub fn get_closest_package_json( - &self, - referrer: &ModuleSpecifier, - ) -> Result>, ClosestPkgJsonError> { - self.node_resolver.get_closest_package_json(referrer) - } - - pub fn resolve_if_for_npm_pkg( - &self, - specifier: &str, - referrer: &ModuleSpecifier, - mode: NodeResolutionMode, - ) -> Result, AnyError> { - let resolution_result = self.resolve(specifier, referrer, mode); - match resolution_result { - Ok(res) => Ok(Some(res)), - Err(err) => { - let err = err.into_kind(); - match err { - NodeResolveErrorKind::RelativeJoin(_) - | NodeResolveErrorKind::PackageImportsResolve(_) - | NodeResolveErrorKind::UnsupportedEsmUrlScheme(_) - | NodeResolveErrorKind::DataUrlReferrer(_) - | NodeResolveErrorKind::TypesNotFound(_) - | NodeResolveErrorKind::FinalizeResolution(_) - | NodeResolveErrorKind::UrlToNodeResolution(_) => Err(err.into()), - NodeResolveErrorKind::PackageResolve(err) => { - let err = err.into_kind(); - match err { - PackageResolveErrorKind::ClosestPkgJson(_) - | PackageResolveErrorKind::InvalidModuleSpecifier(_) - | PackageResolveErrorKind::ExportsResolve(_) - | PackageResolveErrorKind::SubpathResolve(_) => Err(err.into()), - PackageResolveErrorKind::PackageFolderResolve(err) => { - match err.as_kind() { - PackageFolderResolveErrorKind::Io( - PackageFolderResolveIoError { package_name, .. }, - ) - | PackageFolderResolveErrorKind::PackageNotFound( - PackageNotFoundError { package_name, .. }, - ) => { - if self.in_npm_package(referrer) { - return Err(err.into()); - } - if let Some(byonm_npm_resolver) = - self.npm_resolver.as_byonm() - { - if byonm_npm_resolver - .find_ancestor_package_json_with_dep( - package_name, - referrer, - ) - .is_some() - { - return Err(anyhow!( - concat!( - "Could not resolve \"{}\", but found it in a package.json. ", - "Deno expects the node_modules/ directory to be up to date. ", - "Did you forget to run `deno install`?" - ), - specifier - )); - } - } - Ok(None) - } - PackageFolderResolveErrorKind::ReferrerNotFound(_) => { - if self.in_npm_package(referrer) { - return Err(err.into()); - } - Ok(None) - } - } - } - } - } - } - } - } - } - - pub fn resolve( - &self, - specifier: &str, - referrer: &ModuleSpecifier, - mode: NodeResolutionMode, - ) -> Result { - let referrer_kind = if self.cjs_resolutions.contains(referrer) { - NodeModuleKind::Cjs - } else { - NodeModuleKind::Esm - }; - - let res = - self - .node_resolver - .resolve(specifier, referrer, referrer_kind, mode)?; - Ok(self.handle_node_resolution(res)) - } +#[derive(Debug, Clone)] +pub struct CliDenoResolverFs(pub Arc); - pub fn resolve_req_reference( +impl deno_resolver::fs::DenoResolverFs for CliDenoResolverFs { + fn read_to_string_lossy( &self, - req_ref: &NpmPackageReqReference, - referrer: &ModuleSpecifier, - mode: NodeResolutionMode, - ) -> Result { - self.resolve_req_with_sub_path( - req_ref.req(), - req_ref.sub_path(), - referrer, - mode, - ) + path: &Path, + ) -> std::io::Result> { + self + .0 + .read_text_file_lossy_sync(path, None) + .map_err(|e| e.into_io_error()) } - pub fn resolve_req_with_sub_path( - &self, - req: &PackageReq, - sub_path: Option<&str>, - referrer: &ModuleSpecifier, - mode: NodeResolutionMode, - ) -> Result { - let package_folder = self - .npm_resolver - .resolve_pkg_folder_from_deno_module_req(req, referrer)?; - let resolution_result = self.resolve_package_sub_path_from_deno_module( - &package_folder, - sub_path, - Some(referrer), - mode, - ); - match resolution_result { - Ok(resolution) => Ok(resolution), - Err(err) => { - if self.npm_resolver.as_byonm().is_some() { - let package_json_path = package_folder.join("package.json"); - if !self.fs.exists_sync(&package_json_path) { - return Err(anyhow!( - "Could not find '{}'. Deno expects the node_modules/ directory to be up to date. Did you forget to run `deno install`?", - package_json_path.display(), - )); - } - } - Err(err) - } - } + fn realpath_sync(&self, path: &Path) -> std::io::Result { + self.0.realpath_sync(path).map_err(|e| e.into_io_error()) } - pub fn resolve_package_sub_path_from_deno_module( - &self, - package_folder: &Path, - sub_path: Option<&str>, - maybe_referrer: Option<&ModuleSpecifier>, - mode: NodeResolutionMode, - ) -> Result { - let res = self - .node_resolver - .resolve_package_subpath_from_deno_module( - package_folder, - sub_path, - maybe_referrer, - mode, - )?; - Ok(self.handle_node_resolution(res)) + fn exists_sync(&self, path: &Path) -> bool { + self.0.exists_sync(path) } - pub fn handle_if_in_node_modules( - &self, - specifier: &ModuleSpecifier, - ) -> Result, AnyError> { - // skip canonicalizing if we definitely know it's unnecessary - if specifier.scheme() == "file" - && specifier.path().contains("/node_modules/") - { - // Specifiers in the node_modules directory are canonicalized - // so canoncalize then check if it's in the node_modules directory. - // If so, check if we need to store this specifier as being a CJS - // resolution. - let specifier = - crate::node::resolve_specifier_into_node_modules(specifier); - if self.in_npm_package(&specifier) { - let resolution = - self.node_resolver.url_to_node_resolution(specifier)?; - if let NodeResolution::CommonJs(specifier) = &resolution { - self.cjs_resolutions.insert(specifier.clone()); - } - return Ok(Some(resolution.into_url())); - } - } - - Ok(None) + fn is_dir_sync(&self, path: &Path) -> bool { + self.0.is_dir_sync(path) } - pub fn url_to_node_resolution( + fn read_dir_sync( &self, - specifier: ModuleSpecifier, - ) -> Result { - self.node_resolver.url_to_node_resolution(specifier) + dir_path: &Path, + ) -> std::io::Result> { + self + .0 + .read_dir_sync(dir_path) + .map(|entries| { + entries + .into_iter() + .map(|e| deno_resolver::fs::DirEntry { + name: e.name, + is_file: e.is_file, + is_directory: e.is_directory, + }) + .collect::>() + }) + .map_err(|err| err.into_io_error()) } +} - fn handle_node_resolution( - &self, - resolution: NodeResolution, - ) -> NodeResolution { - if let NodeResolution::CommonJs(specifier) = &resolution { - // remember that this was a common js resolution - self.cjs_resolutions.insert(specifier.clone()); - } - resolution - } +#[derive(Debug, Error)] +#[error("{media_type} files are not supported in npm packages: {specifier}")] +pub struct NotSupportedKindInNpmError { + pub media_type: MediaType, + pub specifier: Url, } +// todo(dsherret): move to module_loader.rs (it seems to be here due to use in standalone) #[derive(Clone)] pub struct NpmModuleLoader { - cjs_resolutions: Arc, - node_code_translator: Arc, + cjs_tracker: Arc, fs: Arc, - node_resolver: Arc, + node_code_translator: Arc, } impl NpmModuleLoader { pub fn new( - cjs_resolutions: Arc, - node_code_translator: Arc, + cjs_tracker: Arc, fs: Arc, - node_resolver: Arc, + node_code_translator: Arc, ) -> Self { Self { - cjs_resolutions, + cjs_tracker, node_code_translator, fs, - node_resolver, - } - } - - pub async fn load_if_in_npm_package( - &self, - specifier: &ModuleSpecifier, - maybe_referrer: Option<&ModuleSpecifier>, - ) -> Option> { - if self.node_resolver.in_npm_package(specifier) - || (specifier.scheme() == "file" && specifier.path().ends_with(".cjs")) - { - Some(self.load(specifier, maybe_referrer).await) - } else { - None } } @@ -378,27 +175,33 @@ impl NpmModuleLoader { } })?; - let code = if self.cjs_resolutions.contains(specifier) - || (specifier.scheme() == "file" && specifier.path().ends_with(".cjs")) - { + let media_type = MediaType::from_specifier(specifier); + if media_type.is_emittable() { + return Err(AnyError::from(NotSupportedKindInNpmError { + media_type, + specifier: specifier.clone(), + })); + } + + let code = if self.cjs_tracker.is_maybe_cjs(specifier, media_type)? { // translate cjs to esm if it's cjs and inject node globals - let code = match String::from_utf8_lossy(&code) { - Cow::Owned(code) => code, - // SAFETY: `String::from_utf8_lossy` guarantees that the result is valid - // UTF-8 if `Cow::Borrowed` is returned. - Cow::Borrowed(_) => unsafe { String::from_utf8_unchecked(code) }, - }; + let code = from_utf8_lossy_cow(code); ModuleSourceCode::String( self .node_code_translator .translate_cjs_to_esm(specifier, Some(code)) .await? + .into_owned() .into(), ) } else { // esm and json code is untouched - ModuleSourceCode::Bytes(code.into_boxed_slice().into()) + ModuleSourceCode::Bytes(match code { + Cow::Owned(bytes) => bytes.into_boxed_slice().into(), + Cow::Borrowed(bytes) => bytes.into(), + }) }; + Ok(ModuleCodeStringSource { code, found_url: specifier.clone(), @@ -407,78 +210,36 @@ impl NpmModuleLoader { } } -/// Keeps track of what module specifiers were resolved as CJS. -#[derive(Debug, Default)] -pub struct CjsResolutionStore(DashSet); - -impl CjsResolutionStore { - pub fn contains(&self, specifier: &ModuleSpecifier) -> bool { - self.0.contains(specifier) - } - - pub fn insert(&self, specifier: ModuleSpecifier) { - self.0.insert(specifier); - } +pub struct CliResolverOptions { + pub deno_resolver: Arc, + pub npm_resolver: Option>, + pub bare_node_builtins_enabled: bool, } /// A resolver that takes care of resolution, taking into account loaded /// import map, JSX settings. #[derive(Debug)] -pub struct CliGraphResolver { - node_resolver: Option>, +pub struct CliResolver { + deno_resolver: Arc, npm_resolver: Option>, - sloppy_imports_resolver: Option>, - workspace_resolver: Arc, - maybe_default_jsx_import_source: Option, - maybe_default_jsx_import_source_types: Option, - maybe_jsx_import_source_module: Option, - maybe_vendor_specifier: Option, found_package_json_dep_flag: AtomicFlag, bare_node_builtins_enabled: bool, warned_pkgs: DashSet, } -pub struct CliGraphResolverOptions<'a> { - pub node_resolver: Option>, - pub npm_resolver: Option>, - pub sloppy_imports_resolver: Option>, - pub workspace_resolver: Arc, - pub bare_node_builtins_enabled: bool, - pub maybe_jsx_import_source_config: Option, - pub maybe_vendor_dir: Option<&'a PathBuf>, -} - -impl CliGraphResolver { - pub fn new(options: CliGraphResolverOptions) -> Self { +impl CliResolver { + pub fn new(options: CliResolverOptions) -> Self { Self { - node_resolver: options.node_resolver, + deno_resolver: options.deno_resolver, npm_resolver: options.npm_resolver, - sloppy_imports_resolver: options.sloppy_imports_resolver, - workspace_resolver: options.workspace_resolver, - maybe_default_jsx_import_source: options - .maybe_jsx_import_source_config - .as_ref() - .and_then(|c| c.default_specifier.clone()), - maybe_default_jsx_import_source_types: options - .maybe_jsx_import_source_config - .as_ref() - .and_then(|c| c.default_types_specifier.clone()), - maybe_jsx_import_source_module: options - .maybe_jsx_import_source_config - .map(|c| c.module), - maybe_vendor_specifier: options - .maybe_vendor_dir - .and_then(|v| ModuleSpecifier::from_directory_path(v).ok()), found_package_json_dep_flag: Default::default(), bare_node_builtins_enabled: options.bare_node_builtins_enabled, warned_pkgs: Default::default(), } } - pub fn as_graph_resolver(&self) -> &dyn Resolver { - self - } - + // todo(dsherret): move this off CliResolver as CliResolver is acting + // like a factory by doing this (it's beyond its responsibility) pub fn create_graph_npm_resolver(&self) -> WorkerCliNpmGraphResolver { WorkerCliNpmGraphResolver { npm_resolver: self.npm_resolver.as_ref(), @@ -486,241 +247,55 @@ impl CliGraphResolver { bare_node_builtins_enabled: self.bare_node_builtins_enabled, } } -} - -impl Resolver for CliGraphResolver { - fn default_jsx_import_source(&self) -> Option { - self.maybe_default_jsx_import_source.clone() - } - - fn default_jsx_import_source_types(&self) -> Option { - self.maybe_default_jsx_import_source_types.clone() - } - - fn jsx_import_source_module(&self) -> &str { - self - .maybe_jsx_import_source_module - .as_deref() - .unwrap_or(DEFAULT_JSX_IMPORT_SOURCE_MODULE) - } - fn resolve( + pub fn resolve( &self, raw_specifier: &str, - referrer_range: &deno_graph::Range, - mode: ResolutionMode, + referrer: &ModuleSpecifier, + referrer_range_start: deno_graph::Position, + resolution_mode: ResolutionMode, + resolution_kind: NodeResolutionKind, ) -> Result { - fn to_node_mode(mode: ResolutionMode) -> NodeResolutionMode { - match mode { - ResolutionMode::Execution => NodeResolutionMode::Execution, - ResolutionMode::Types => NodeResolutionMode::Types, - } - } - - let referrer = &referrer_range.specifier; + let resolution = self + .deno_resolver + .resolve(raw_specifier, referrer, resolution_mode, resolution_kind) + .map_err(|err| match err.into_kind() { + deno_resolver::DenoResolveErrorKind::MappedResolution( + mapped_resolution_error, + ) => match mapped_resolution_error { + MappedResolutionError::Specifier(e) => ResolveError::Specifier(e), + // deno_graph checks specifically for an ImportMapError + MappedResolutionError::ImportMap(e) => ResolveError::Other(e.into()), + err => ResolveError::Other(err.into()), + }, + err => ResolveError::Other(err.into()), + })?; - // Use node resolution if we're in an npm package - if let Some(node_resolver) = self.node_resolver.as_ref() { - if referrer.scheme() == "file" && node_resolver.in_npm_package(referrer) { - return node_resolver - .resolve(raw_specifier, referrer, to_node_mode(mode)) - .map(|res| res.into_url()) - .map_err(|e| ResolveError::Other(e.into())); - } + if resolution.found_package_json_dep { + // mark that we need to do an "npm install" later + self.found_package_json_dep_flag.raise(); } - // Attempt to resolve with the workspace resolver - let result: Result<_, ResolveError> = self - .workspace_resolver - .resolve(raw_specifier, referrer) - .map_err(|err| match err { - MappedResolutionError::Specifier(err) => ResolveError::Specifier(err), - MappedResolutionError::ImportMap(err) => { - ResolveError::Other(err.into()) - } - MappedResolutionError::Workspace(err) => { - ResolveError::Other(err.into()) - } - }); - let result = match result { - Ok(resolution) => match resolution { - MappedResolution::Normal { - specifier, - maybe_diagnostic, - } - | MappedResolution::ImportMap { - specifier, - maybe_diagnostic, - } => { - if let Some(diagnostic) = maybe_diagnostic { - match &*diagnostic { - MappedResolutionDiagnostic::ConstraintNotMatchedLocalVersion { reference, .. } => { - if self.warned_pkgs.insert(reference.req().clone()) { - log::warn!("{} {}\n at {}", colors::yellow("Warning"), diagnostic, referrer_range); - } - } - } - } - // do sloppy imports resolution if enabled - if let Some(sloppy_imports_resolver) = &self.sloppy_imports_resolver { - Ok( - sloppy_imports_resolver - .resolve(&specifier, mode) - .map(|s| s.into_specifier()) - .unwrap_or(specifier), - ) - } else { - Ok(specifier) - } - } - MappedResolution::WorkspaceJsrPackage { specifier, .. } => { - Ok(specifier) - } - MappedResolution::WorkspaceNpmPackage { - target_pkg_json: pkg_json, - sub_path, - .. - } => self - .node_resolver - .as_ref() - .unwrap() - .resolve_package_sub_path_from_deno_module( - pkg_json.dir_path(), - sub_path.as_deref(), - Some(referrer), - to_node_mode(mode), - ) - .map_err(ResolveError::Other) - .map(|res| res.into_url()), - MappedResolution::PackageJson { - dep_result, - alias, - sub_path, + if let Some(diagnostic) = resolution.maybe_diagnostic { + match &*diagnostic { + MappedResolutionDiagnostic::ConstraintNotMatchedLocalVersion { + reference, .. } => { - // found a specifier in the package.json, so mark that - // we need to do an "npm install" later - self.found_package_json_dep_flag.raise(); - - dep_result - .as_ref() - .map_err(|e| ResolveError::Other(e.clone().into())) - .and_then(|dep| match dep { - PackageJsonDepValue::Req(req) => { - ModuleSpecifier::parse(&format!( - "npm:{}{}", - req, - sub_path.map(|s| format!("/{}", s)).unwrap_or_default() - )) - .map_err(|e| ResolveError::Other(e.into())) - } - PackageJsonDepValue::Workspace(version_req) => self - .workspace_resolver - .resolve_workspace_pkg_json_folder_for_pkg_json_dep( - alias, - version_req, - ) - .map_err(|e| ResolveError::Other(e.into())) - .and_then(|pkg_folder| { - Ok( - self - .node_resolver - .as_ref() - .unwrap() - .resolve_package_sub_path_from_deno_module( - pkg_folder, - sub_path.as_deref(), - Some(referrer), - to_node_mode(mode), - )? - .into_url(), - ) - }), - }) - } - }, - Err(err) => Err(err), - }; - - // When the user is vendoring, don't allow them to import directly from the vendor/ directory - // as it might cause them confusion or duplicate dependencies. Additionally, this folder has - // special treatment in the language server so it will definitely cause issues/confusion there - // if they do this. - if let Some(vendor_specifier) = &self.maybe_vendor_specifier { - if let Ok(specifier) = &result { - if specifier.as_str().starts_with(vendor_specifier.as_str()) { - return Err(ResolveError::Other(anyhow!("Importing from the vendor directory is not permitted. Use a remote specifier instead or disable vendoring."))); - } - } - } - - let Some(node_resolver) = &self.node_resolver else { - return result; - }; - - let is_byonm = self - .npm_resolver - .as_ref() - .is_some_and(|r| r.as_byonm().is_some()); - match result { - Ok(specifier) => { - if let Ok(npm_req_ref) = - NpmPackageReqReference::from_specifier(&specifier) - { - // check if the npm specifier resolves to a workspace member - if let Some(pkg_folder) = self - .workspace_resolver - .resolve_workspace_pkg_json_folder_for_npm_specifier( - npm_req_ref.req(), - ) - { - return Ok( - node_resolver - .resolve_package_sub_path_from_deno_module( - pkg_folder, - npm_req_ref.sub_path(), - Some(referrer), - to_node_mode(mode), - )? - .into_url(), + if self.warned_pkgs.insert(reference.req().clone()) { + log::warn!( + "{} {}\n at {}:{}", + colors::yellow("Warning"), + diagnostic, + referrer, + referrer_range_start, ); } - - // do npm resolution for byonm - if is_byonm { - return node_resolver - .resolve_req_reference(&npm_req_ref, referrer, to_node_mode(mode)) - .map(|res| res.into_url()) - .map_err(|err| err.into()); - } - } - - Ok(match node_resolver.handle_if_in_node_modules(&specifier)? { - Some(specifier) => specifier, - None => specifier, - }) - } - Err(err) => { - // If byonm, check if the bare specifier resolves to an npm package - if is_byonm && referrer.scheme() == "file" { - let maybe_resolution = node_resolver - .resolve_if_for_npm_pkg(raw_specifier, referrer, to_node_mode(mode)) - .map_err(ResolveError::Other)?; - if let Some(res) = maybe_resolution { - match res { - NodeResolution::Esm(url) | NodeResolution::CommonJs(url) => { - return Ok(url) - } - NodeResolution::BuiltIn(_) => { - // don't resolve bare specifiers for built-in modules via node resolution - } - } - } } - - Err(err) } } + + Ok(resolution.url) } } @@ -754,13 +329,10 @@ impl<'a> deno_graph::source::NpmResolver for WorkerCliNpmGraphResolver<'a> { module_name: &str, range: &deno_graph::Range, ) { - let deno_graph::Range { - start, specifier, .. - } = range; - let line = start.line + 1; - let column = start.character + 1; + let start = range.range.start; + let specifier = &range.specifier; if !*DENO_DISABLE_PEDANTIC_NODE_WARNINGS { - log::warn!("{} Resolving \"{module_name}\" as \"node:{module_name}\" at {specifier}:{line}:{column}. If you want to use a built-in Node module, add a \"node:\" prefix.", colors::yellow("Warning")) + log::warn!("{} Resolving \"{module_name}\" as \"node:{module_name}\" at {specifier}:{start}. If you want to use a built-in Node module, add a \"node:\" prefix.", colors::yellow("Warning")) } } @@ -847,96 +419,18 @@ impl<'a> deno_graph::source::NpmResolver for WorkerCliNpmGraphResolver<'a> { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SloppyImportsFsEntry { - File, - Dir, -} - -impl SloppyImportsFsEntry { - pub fn from_fs_stat( - stat: &deno_runtime::deno_io::fs::FsStat, - ) -> Option { - if stat.is_file { - Some(SloppyImportsFsEntry::File) - } else if stat.is_directory { - Some(SloppyImportsFsEntry::Dir) - } else { - None - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum SloppyImportsResolution { - /// Ex. `./file.js` to `./file.ts` - JsToTs(ModuleSpecifier), - /// Ex. `./file` to `./file.ts` - NoExtension(ModuleSpecifier), - /// Ex. `./dir` to `./dir/index.ts` - Directory(ModuleSpecifier), -} - -impl SloppyImportsResolution { - pub fn as_specifier(&self) -> &ModuleSpecifier { - match self { - Self::JsToTs(specifier) => specifier, - Self::NoExtension(specifier) => specifier, - Self::Directory(specifier) => specifier, - } - } - - pub fn into_specifier(self) -> ModuleSpecifier { - match self { - Self::JsToTs(specifier) => specifier, - Self::NoExtension(specifier) => specifier, - Self::Directory(specifier) => specifier, - } - } - - pub fn as_suggestion_message(&self) -> String { - format!("Maybe {}", self.as_base_message()) - } - - pub fn as_quick_fix_message(&self) -> String { - let message = self.as_base_message(); - let mut chars = message.chars(); - format!( - "{}{}.", - chars.next().unwrap().to_uppercase(), - chars.as_str() - ) - } - - fn as_base_message(&self) -> String { - match self { - SloppyImportsResolution::JsToTs(specifier) => { - let media_type = MediaType::from_specifier(specifier); - format!("change the extension to '{}'", media_type.as_ts_extension()) - } - SloppyImportsResolution::NoExtension(specifier) => { - let media_type = MediaType::from_specifier(specifier); - format!("add a '{}' extension", media_type.as_ts_extension()) - } - SloppyImportsResolution::Directory(specifier) => { - let file_name = specifier - .path() - .rsplit_once('/') - .map(|(_, file_name)| file_name) - .unwrap_or(specifier.path()); - format!("specify path to '{}' file in directory instead", file_name) - } - } - } -} - #[derive(Debug)] -pub struct SloppyImportsResolver { - fs: Arc, - cache: Option>>, +pub struct SloppyImportsCachedFs { + fs: Arc, + cache: Option< + DashMap< + PathBuf, + Option, + >, + >, } -impl SloppyImportsResolver { +impl SloppyImportsCachedFs { pub fn new(fs: Arc) -> Self { Self { fs, @@ -947,409 +441,34 @@ impl SloppyImportsResolver { pub fn new_without_stat_cache(fs: Arc) -> Self { Self { fs, cache: None } } +} - pub fn resolve( +impl deno_resolver::sloppy_imports::SloppyImportResolverFs + for SloppyImportsCachedFs +{ + fn stat_sync( &self, - specifier: &ModuleSpecifier, - mode: ResolutionMode, - ) -> Option { - fn path_without_ext( - path: &Path, - media_type: MediaType, - ) -> Option> { - let old_path_str = path.to_string_lossy(); - match media_type { - MediaType::Unknown => Some(old_path_str), - _ => old_path_str - .strip_suffix(media_type.as_ts_extension()) - .map(|s| Cow::Owned(s.to_string())), - } - } - - fn media_types_to_paths( - path_no_ext: &str, - original_media_type: MediaType, - probe_media_type_types: Vec, - reason: SloppyImportsResolutionReason, - ) -> Vec<(PathBuf, SloppyImportsResolutionReason)> { - probe_media_type_types - .into_iter() - .filter(|media_type| *media_type != original_media_type) - .map(|media_type| { - ( - PathBuf::from(format!( - "{}{}", - path_no_ext, - media_type.as_ts_extension() - )), - reason, - ) - }) - .collect::>() - } - - if specifier.scheme() != "file" { - return None; - } - - let path = specifier_to_file_path(specifier).ok()?; - - #[derive(Clone, Copy)] - enum SloppyImportsResolutionReason { - JsToTs, - NoExtension, - Directory, - } - - let probe_paths: Vec<(PathBuf, SloppyImportsResolutionReason)> = - match self.stat_sync(&path) { - Some(SloppyImportsFsEntry::File) => { - if mode.is_types() { - let media_type = MediaType::from_specifier(specifier); - // attempt to resolve the .d.ts file before the .js file - let probe_media_type_types = match media_type { - MediaType::JavaScript => { - vec![(MediaType::Dts), MediaType::JavaScript] - } - MediaType::Mjs => { - vec![MediaType::Dmts, MediaType::Dts, MediaType::Mjs] - } - MediaType::Cjs => { - vec![MediaType::Dcts, MediaType::Dts, MediaType::Cjs] - } - _ => return None, - }; - let path_no_ext = path_without_ext(&path, media_type)?; - media_types_to_paths( - &path_no_ext, - media_type, - probe_media_type_types, - SloppyImportsResolutionReason::JsToTs, - ) - } else { - return None; - } - } - entry @ None | entry @ Some(SloppyImportsFsEntry::Dir) => { - let media_type = MediaType::from_specifier(specifier); - let probe_media_type_types = match media_type { - MediaType::JavaScript => ( - if mode.is_types() { - vec![MediaType::TypeScript, MediaType::Tsx, MediaType::Dts] - } else { - vec![MediaType::TypeScript, MediaType::Tsx] - }, - SloppyImportsResolutionReason::JsToTs, - ), - MediaType::Jsx => { - (vec![MediaType::Tsx], SloppyImportsResolutionReason::JsToTs) - } - MediaType::Mjs => ( - if mode.is_types() { - vec![MediaType::Mts, MediaType::Dmts, MediaType::Dts] - } else { - vec![MediaType::Mts] - }, - SloppyImportsResolutionReason::JsToTs, - ), - MediaType::Cjs => ( - if mode.is_types() { - vec![MediaType::Cts, MediaType::Dcts, MediaType::Dts] - } else { - vec![MediaType::Cts] - }, - SloppyImportsResolutionReason::JsToTs, - ), - MediaType::TypeScript - | MediaType::Mts - | MediaType::Cts - | MediaType::Dts - | MediaType::Dmts - | MediaType::Dcts - | MediaType::Tsx - | MediaType::Json - | MediaType::Wasm - | MediaType::TsBuildInfo - | MediaType::SourceMap => { - return None; - } - MediaType::Unknown => ( - if mode.is_types() { - vec![ - MediaType::TypeScript, - MediaType::Tsx, - MediaType::Mts, - MediaType::Dts, - MediaType::Dmts, - MediaType::Dcts, - MediaType::JavaScript, - MediaType::Jsx, - MediaType::Mjs, - ] - } else { - vec![ - MediaType::TypeScript, - MediaType::JavaScript, - MediaType::Tsx, - MediaType::Jsx, - MediaType::Mts, - MediaType::Mjs, - ] - }, - SloppyImportsResolutionReason::NoExtension, - ), - }; - let mut probe_paths = match path_without_ext(&path, media_type) { - Some(path_no_ext) => media_types_to_paths( - &path_no_ext, - media_type, - probe_media_type_types.0, - probe_media_type_types.1, - ), - None => vec![], - }; - - if matches!(entry, Some(SloppyImportsFsEntry::Dir)) { - // try to resolve at the index file - if mode.is_types() { - probe_paths.push(( - path.join("index.ts"), - SloppyImportsResolutionReason::Directory, - )); - - probe_paths.push(( - path.join("index.mts"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.d.ts"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.d.mts"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.js"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.mjs"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.tsx"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.jsx"), - SloppyImportsResolutionReason::Directory, - )); - } else { - probe_paths.push(( - path.join("index.ts"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.mts"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.tsx"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.js"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.mjs"), - SloppyImportsResolutionReason::Directory, - )); - probe_paths.push(( - path.join("index.jsx"), - SloppyImportsResolutionReason::Directory, - )); - } - } - if probe_paths.is_empty() { - return None; - } - probe_paths - } - }; - - for (probe_path, reason) in probe_paths { - if self.stat_sync(&probe_path) == Some(SloppyImportsFsEntry::File) { - if let Ok(specifier) = ModuleSpecifier::from_file_path(probe_path) { - match reason { - SloppyImportsResolutionReason::JsToTs => { - return Some(SloppyImportsResolution::JsToTs(specifier)); - } - SloppyImportsResolutionReason::NoExtension => { - return Some(SloppyImportsResolution::NoExtension(specifier)); - } - SloppyImportsResolutionReason::Directory => { - return Some(SloppyImportsResolution::Directory(specifier)); - } - } - } - } - } - - None - } - - fn stat_sync(&self, path: &Path) -> Option { + path: &Path, + ) -> Option { if let Some(cache) = &self.cache { if let Some(entry) = cache.get(path) { return *entry; } } - let entry = self - .fs - .stat_sync(path) - .ok() - .and_then(|stat| SloppyImportsFsEntry::from_fs_stat(&stat)); + let entry = self.fs.stat_sync(path).ok().and_then(|stat| { + if stat.is_file { + Some(deno_resolver::sloppy_imports::SloppyImportsFsEntry::File) + } else if stat.is_directory { + Some(deno_resolver::sloppy_imports::SloppyImportsFsEntry::Dir) + } else { + None + } + }); + if let Some(cache) = &self.cache { cache.insert(path.to_owned(), entry); } entry } } - -#[cfg(test)] -mod test { - use test_util::TestContext; - - use super::*; - - #[test] - fn test_unstable_sloppy_imports() { - fn resolve(specifier: &ModuleSpecifier) -> Option { - resolve_with_mode(specifier, ResolutionMode::Execution) - } - - fn resolve_types( - specifier: &ModuleSpecifier, - ) -> Option { - resolve_with_mode(specifier, ResolutionMode::Types) - } - - fn resolve_with_mode( - specifier: &ModuleSpecifier, - mode: ResolutionMode, - ) -> Option { - SloppyImportsResolver::new(Arc::new(deno_fs::RealFs)) - .resolve(specifier, mode) - } - - let context = TestContext::default(); - let temp_dir = context.temp_dir().path(); - - // scenarios like resolving ./example.js to ./example.ts - for (ext_from, ext_to) in [("js", "ts"), ("js", "tsx"), ("mjs", "mts")] { - let ts_file = temp_dir.join(format!("file.{}", ext_to)); - ts_file.write(""); - assert_eq!(resolve(&ts_file.url_file()), None); - assert_eq!( - resolve( - &temp_dir - .url_dir() - .join(&format!("file.{}", ext_from)) - .unwrap() - ), - Some(SloppyImportsResolution::JsToTs(ts_file.url_file())), - ); - ts_file.remove_file(); - } - - // no extension scenarios - for ext in ["js", "ts", "js", "tsx", "jsx", "mjs", "mts"] { - let file = temp_dir.join(format!("file.{}", ext)); - file.write(""); - assert_eq!( - resolve( - &temp_dir - .url_dir() - .join("file") // no ext - .unwrap() - ), - Some(SloppyImportsResolution::NoExtension(file.url_file())) - ); - file.remove_file(); - } - - // .ts and .js exists, .js specified (goes to specified) - { - let ts_file = temp_dir.join("file.ts"); - ts_file.write(""); - let js_file = temp_dir.join("file.js"); - js_file.write(""); - assert_eq!(resolve(&js_file.url_file()), None); - } - - // only js exists, .js specified - { - let js_only_file = temp_dir.join("js_only.js"); - js_only_file.write(""); - assert_eq!(resolve(&js_only_file.url_file()), None); - assert_eq!(resolve_types(&js_only_file.url_file()), None); - } - - // resolving a directory to an index file - { - let routes_dir = temp_dir.join("routes"); - routes_dir.create_dir_all(); - let index_file = routes_dir.join("index.ts"); - index_file.write(""); - assert_eq!( - resolve(&routes_dir.url_file()), - Some(SloppyImportsResolution::Directory(index_file.url_file())), - ); - } - - // both a directory and a file with specifier is present - { - let api_dir = temp_dir.join("api"); - api_dir.create_dir_all(); - let bar_file = api_dir.join("bar.ts"); - bar_file.write(""); - let api_file = temp_dir.join("api.ts"); - api_file.write(""); - assert_eq!( - resolve(&api_dir.url_file()), - Some(SloppyImportsResolution::NoExtension(api_file.url_file())), - ); - } - } - - #[test] - fn test_sloppy_import_resolution_suggestion_message() { - // directory - assert_eq!( - SloppyImportsResolution::Directory( - ModuleSpecifier::parse("file:///dir/index.js").unwrap() - ) - .as_suggestion_message(), - "Maybe specify path to 'index.js' file in directory instead" - ); - // no ext - assert_eq!( - SloppyImportsResolution::NoExtension( - ModuleSpecifier::parse("file:///dir/index.mjs").unwrap() - ) - .as_suggestion_message(), - "Maybe add a '.mjs' extension" - ); - // js to ts - assert_eq!( - SloppyImportsResolution::JsToTs( - ModuleSpecifier::parse("file:///dir/index.mts").unwrap() - ) - .as_suggestion_message(), - "Maybe change the extension to '.mts'" - ); - } -} diff --git a/cli/schemas/config-file.v1.json b/cli/schemas/config-file.v1.json index af18e6f21c2047..a64cb2ff655d42 100644 --- a/cli/schemas/config-file.v1.json +++ b/cli/schemas/config-file.v1.json @@ -291,7 +291,7 @@ "type": "array", "description": "List of tag names that will be run. Empty list disables all tags and will only use rules from `include`.", "items": { - "type": "string" + "$ref": "https://raw.githubusercontent.com/denoland/deno_lint/main/schemas/tags.v1.json" }, "minItems": 0, "uniqueItems": true @@ -300,7 +300,7 @@ "type": "array", "description": "List of rule names that will be excluded from configured tag sets. If the same rule is in `include` it will be run.", "items": { - "type": "string" + "$ref": "https://raw.githubusercontent.com/denoland/deno_lint/main/schemas/rules.v1.json" }, "minItems": 0, "uniqueItems": true @@ -309,7 +309,7 @@ "type": "array", "description": "List of rule names that will be run. Even if the same rule is in `exclude` it will be run.", "items": { - "type": "string" + "$ref": "https://raw.githubusercontent.com/denoland/deno_lint/main/schemas/rules.v1.json" }, "minItems": 0, "uniqueItems": true @@ -431,8 +431,34 @@ "type": "object", "patternProperties": { "^[A-Za-z][A-Za-z0-9_\\-:]*$": { - "type": "string", - "description": "Command to execute for this task name." + "oneOf": [ + { + "type": "string", + "description": "Command to execute for this task name." + }, + { + "type": "object", + "description": "A definition of a task to execute", + "properties": { + "description": { + "type": "string", + "description": "Description of a task that will be shown when running `deno task` without a task name" + }, + "command": { + "type": "string", + "required": true, + "description": "The task to execute" + }, + "dependencies": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Tasks that should be executed before this task" + } + } + } + ] } }, "additionalProperties": false @@ -528,11 +554,15 @@ "bare-node-builtins", "byonm", "cron", + "detect-cjs", "ffi", "fs", + "fmt-component", + "fmt-sql", "http", "kv", "net", + "node-globals", "sloppy-imports", "temporal", "unsafe-proto", diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 1e84e13986c067..632f27da6fdfca 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -4,19 +4,26 @@ use std::borrow::Cow; use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::VecDeque; +use std::env; use std::env::current_exe; use std::ffi::OsString; use std::fs; use std::fs::File; use std::future::Future; +use std::io::ErrorKind; use std::io::Read; use std::io::Seek; use std::io::SeekFrom; use std::io::Write; +use std::ops::Range; +use std::path::Component; use std::path::Path; use std::path::PathBuf; use std::process::Command; +use std::sync::Arc; +use deno_ast::MediaType; +use deno_ast::ModuleKind; use deno_ast::ModuleSpecifier; use deno_config::workspace::PackageJsonDepResolution; use deno_config::workspace::ResolverWorkspaceJsrPackage; @@ -30,13 +37,23 @@ use deno_core::futures::AsyncReadExt; use deno_core::futures::AsyncSeekExt; use deno_core::serde_json; use deno_core::url::Url; +use deno_graph::source::RealFileSystem; +use deno_graph::ModuleGraph; +use deno_npm::resolution::SerializedNpmResolutionSnapshot; +use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; +use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; +use deno_npm::NpmPackageId; use deno_npm::NpmSystemInfo; +use deno_runtime::deno_fs; +use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_fs::RealFs; +use deno_runtime::deno_io::fs::FsError; use deno_runtime::deno_node::PackageJson; use deno_semver::npm::NpmVersionReqParseError; use deno_semver::package::PackageReq; use deno_semver::Version; use deno_semver::VersionReqSpecifierParseError; -use eszip::EszipRelativeFileBaseUrl; +use deno_telemetry::OtelConfig; use indexmap::IndexMap; use log::Level; use serde::Deserialize; @@ -49,10 +66,13 @@ use crate::args::NpmInstallDepsProvider; use crate::args::PermissionFlags; use crate::args::UnstableConfig; use crate::cache::DenoDir; +use crate::cache::FastInsecureHasher; +use crate::emit::Emitter; use crate::file_fetcher::FileFetcher; use crate::http_util::HttpClientProvider; use crate::npm::CliNpmResolver; use crate::npm::InnerCliNpmResolverRef; +use crate::resolver::CjsTracker; use crate::shared::ReleaseChannel; use crate::standalone::virtual_fs::VfsEntry; use crate::util::archive; @@ -60,12 +80,64 @@ use crate::util::fs::canonicalize_path_maybe_not_exists; use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; +use super::file_system::DenoCompileFileSystem; +use super::serialization::deserialize_binary_data_section; +use super::serialization::serialize_binary_data_section; +use super::serialization::DenoCompileModuleData; +use super::serialization::DeserializedDataSection; +use super::serialization::RemoteModulesStore; +use super::serialization::RemoteModulesStoreBuilder; use super::virtual_fs::FileBackedVfs; use super::virtual_fs::VfsBuilder; +use super::virtual_fs::VfsFileSubDataKind; use super::virtual_fs::VfsRoot; use super::virtual_fs::VirtualDirectory; -const MAGIC_TRAILER: &[u8; 8] = b"d3n0l4nd"; +/// A URL that can be designated as the base for relative URLs. +/// +/// After creation, this URL may be used to get the key for a +/// module in the binary. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct StandaloneRelativeFileBaseUrl<'a>(&'a Url); + +impl<'a> From<&'a Url> for StandaloneRelativeFileBaseUrl<'a> { + fn from(url: &'a Url) -> Self { + Self(url) + } +} + +impl<'a> StandaloneRelativeFileBaseUrl<'a> { + pub fn new(url: &'a Url) -> Self { + debug_assert_eq!(url.scheme(), "file"); + Self(url) + } + + /// Gets the module map key of the provided specifier. + /// + /// * Descendant file specifiers will be made relative to the base. + /// * Non-descendant file specifiers will stay as-is (absolute). + /// * Non-file specifiers will stay as-is. + pub fn specifier_key<'b>(&self, target: &'b Url) -> Cow<'b, str> { + if target.scheme() != "file" { + return Cow::Borrowed(target.as_str()); + } + + match self.0.make_relative(target) { + Some(relative) => { + if relative.starts_with("../") { + Cow::Borrowed(target.as_str()) + } else { + Cow::Owned(relative) + } + } + None => Cow::Borrowed(target.as_str()), + } + } + + pub fn inner(&self) -> &Url { + self.0 + } +} #[derive(Deserialize, Serialize)] pub enum NodeModules { @@ -106,6 +178,7 @@ pub struct SerializedWorkspaceResolver { pub struct Metadata { pub argv: Vec, pub seed: Option, + pub code_cache_key: Option, pub permissions: PermissionFlags, pub location: Option, pub v8_flags: Vec, @@ -118,80 +191,27 @@ pub struct Metadata { pub entrypoint_key: String, pub node_modules: Option, pub unstable_config: UnstableConfig, -} - -pub fn load_npm_vfs(root_dir_path: PathBuf) -> Result { - let data = libsui::find_section("d3n0l4nd").unwrap(); - - // We do the first part sync so it can complete quickly - let trailer: [u8; TRAILER_SIZE] = data[0..TRAILER_SIZE].try_into().unwrap(); - let trailer = match Trailer::parse(&trailer)? { - None => panic!("Could not find trailer"), - Some(trailer) => trailer, - }; - let data = &data[TRAILER_SIZE..]; - - let vfs_data = - &data[trailer.npm_vfs_pos as usize..trailer.npm_files_pos as usize]; - let mut dir: VirtualDirectory = serde_json::from_slice(vfs_data)?; - - // align the name of the directory with the root dir - dir.name = root_dir_path - .file_name() - .unwrap() - .to_string_lossy() - .to_string(); - - let fs_root = VfsRoot { - dir, - root_path: root_dir_path, - start_file_offset: trailer.npm_files_pos, - }; - Ok(FileBackedVfs::new(data.to_vec(), fs_root)) + pub otel_config: Option, // None means disabled. } fn write_binary_bytes( mut file_writer: File, original_bin: Vec, metadata: &Metadata, - eszip: eszip::EszipV2, - npm_vfs: Option<&VirtualDirectory>, - npm_files: &Vec>, + npm_snapshot: Option, + remote_modules: &RemoteModulesStoreBuilder, + vfs: VfsBuilder, compile_flags: &CompileFlags, ) -> Result<(), AnyError> { - let metadata = serde_json::to_string(metadata)?.as_bytes().to_vec(); - let npm_vfs = serde_json::to_string(&npm_vfs)?.as_bytes().to_vec(); - let eszip_archive = eszip.into_bytes(); - - let mut writer = Vec::new(); - - // write the trailer, which includes the positions - // of the data blocks in the file - writer.write_all(&{ - let metadata_pos = eszip_archive.len() as u64; - let npm_vfs_pos = metadata_pos + (metadata.len() as u64); - let npm_files_pos = npm_vfs_pos + (npm_vfs.len() as u64); - Trailer { - eszip_pos: 0, - metadata_pos, - npm_vfs_pos, - npm_files_pos, - } - .as_bytes() - })?; - - writer.write_all(&eszip_archive)?; - writer.write_all(&metadata)?; - writer.write_all(&npm_vfs)?; - for file in npm_files { - writer.write_all(file)?; - } + let data_section_bytes = + serialize_binary_data_section(metadata, npm_snapshot, remote_modules, vfs) + .context("Serializing binary data section.")?; let target = compile_flags.resolve_target(); if target.contains("linux") { libsui::Elf::new(&original_bin).append( "d3n0l4nd", - &writer, + &data_section_bytes, &mut file_writer, )?; } else if target.contains("windows") { @@ -201,11 +221,11 @@ fn write_binary_bytes( pe = pe.set_icon(&icon)?; } - pe.write_resource("d3n0l4nd", writer)? + pe.write_resource("d3n0l4nd", data_section_bytes)? .build(&mut file_writer)?; } else if target.contains("darwin") { libsui::Macho::from(original_bin)? - .write_section("d3n0l4nd", writer)? + .write_section("d3n0l4nd", data_section_bytes)? .build_and_sign(&mut file_writer)?; } Ok(()) @@ -221,6 +241,68 @@ pub fn is_standalone_binary(exe_path: &Path) -> bool { || libsui::utils::is_macho(&data) } +pub struct StandaloneData { + pub fs: Arc, + pub metadata: Metadata, + pub modules: StandaloneModules, + pub npm_snapshot: Option, + pub root_path: PathBuf, + pub vfs: Arc, +} + +pub struct StandaloneModules { + remote_modules: RemoteModulesStore, + vfs: Arc, +} + +impl StandaloneModules { + pub fn resolve_specifier<'a>( + &'a self, + specifier: &'a ModuleSpecifier, + ) -> Result, AnyError> { + if specifier.scheme() == "file" { + Ok(Some(specifier)) + } else { + self.remote_modules.resolve_specifier(specifier) + } + } + + pub fn has_file(&self, path: &Path) -> bool { + self.vfs.file_entry(path).is_ok() + } + + pub fn read<'a>( + &'a self, + specifier: &'a ModuleSpecifier, + ) -> Result>, AnyError> { + if specifier.scheme() == "file" { + let path = deno_path_util::url_to_file_path(specifier)?; + let bytes = match self.vfs.file_entry(&path) { + Ok(entry) => self + .vfs + .read_file_all(entry, VfsFileSubDataKind::ModuleGraph)?, + Err(err) if err.kind() == ErrorKind::NotFound => { + match RealFs.read_file_sync(&path, None) { + Ok(bytes) => bytes, + Err(FsError::Io(err)) if err.kind() == ErrorKind::NotFound => { + return Ok(None) + } + Err(err) => return Err(err.into()), + } + } + Err(err) => return Err(err.into()), + }; + Ok(Some(DenoCompileModuleData { + media_type: MediaType::from_specifier(specifier), + specifier, + data: bytes, + })) + } else { + self.remote_modules.read(specifier) + } + } +} + /// This function will try to run this binary as a standalone binary /// produced by `deno compile`. It determines if this is a standalone /// binary by skipping over the trailer width at the end of the file, @@ -228,110 +310,68 @@ pub fn is_standalone_binary(exe_path: &Path) -> bool { /// the bundle is executed. If not, this function exits with `Ok(None)`. pub fn extract_standalone( cli_args: Cow>, -) -> Result< - Option>>, - AnyError, -> { +) -> Result, AnyError> { let Some(data) = libsui::find_section("d3n0l4nd") else { return Ok(None); }; - // We do the first part sync so it can complete quickly - let trailer = match Trailer::parse(&data[0..TRAILER_SIZE])? { + let DeserializedDataSection { + mut metadata, + npm_snapshot, + remote_modules, + mut vfs_dir, + vfs_files_data, + } = match deserialize_binary_data_section(data)? { + Some(data_section) => data_section, None => return Ok(None), - Some(trailer) => trailer, }; + let root_path = { + let maybe_current_exe = std::env::current_exe().ok(); + let current_exe_name = maybe_current_exe + .as_ref() + .and_then(|p| p.file_name()) + .map(|p| p.to_string_lossy()) + // should never happen + .unwrap_or_else(|| Cow::Borrowed("binary")); + std::env::temp_dir().join(format!("deno-compile-{}", current_exe_name)) + }; let cli_args = cli_args.into_owned(); - // If we have an eszip, read it out - Ok(Some(async move { - let bufreader = - deno_core::futures::io::BufReader::new(&data[TRAILER_SIZE..]); - - let (eszip, loader) = eszip::EszipV2::parse(bufreader) - .await - .context("Failed to parse eszip header")?; - - let bufreader = loader.await.context("Failed to parse eszip archive")?; - - let mut metadata = String::new(); - - bufreader - .take(trailer.metadata_len()) - .read_to_string(&mut metadata) - .await - .context("Failed to read metadata from the current executable")?; - - let mut metadata: Metadata = serde_json::from_str(&metadata).unwrap(); - metadata.argv.reserve(cli_args.len() - 1); - for arg in cli_args.into_iter().skip(1) { - metadata.argv.push(arg.into_string().unwrap()); - } - - Ok((metadata, eszip)) - })) -} - -const TRAILER_SIZE: usize = std::mem::size_of::() + 8; // 8 bytes for the magic trailer string - -struct Trailer { - eszip_pos: u64, - metadata_pos: u64, - npm_vfs_pos: u64, - npm_files_pos: u64, -} - -impl Trailer { - pub fn parse(trailer: &[u8]) -> Result, AnyError> { - let (magic_trailer, rest) = trailer.split_at(8); - if magic_trailer != MAGIC_TRAILER { - return Ok(None); - } - - let (eszip_archive_pos, rest) = rest.split_at(8); - let (metadata_pos, rest) = rest.split_at(8); - let (npm_vfs_pos, npm_files_pos) = rest.split_at(8); - let eszip_archive_pos = u64_from_bytes(eszip_archive_pos)?; - let metadata_pos = u64_from_bytes(metadata_pos)?; - let npm_vfs_pos = u64_from_bytes(npm_vfs_pos)?; - let npm_files_pos = u64_from_bytes(npm_files_pos)?; - Ok(Some(Trailer { - eszip_pos: eszip_archive_pos, - metadata_pos, - npm_vfs_pos, - npm_files_pos, - })) - } - - pub fn metadata_len(&self) -> u64 { - self.npm_vfs_pos - self.metadata_pos - } - - pub fn npm_vfs_len(&self) -> u64 { - self.npm_files_pos - self.npm_vfs_pos + metadata.argv.reserve(cli_args.len() - 1); + for arg in cli_args.into_iter().skip(1) { + metadata.argv.push(arg.into_string().unwrap()); } - - pub fn as_bytes(&self) -> Vec { - let mut trailer = MAGIC_TRAILER.to_vec(); - trailer.write_all(&self.eszip_pos.to_be_bytes()).unwrap(); - trailer.write_all(&self.metadata_pos.to_be_bytes()).unwrap(); - trailer.write_all(&self.npm_vfs_pos.to_be_bytes()).unwrap(); - trailer - .write_all(&self.npm_files_pos.to_be_bytes()) - .unwrap(); - trailer - } -} - -fn u64_from_bytes(arr: &[u8]) -> Result { - let fixed_arr: &[u8; 8] = arr - .try_into() - .context("Failed to convert the buffer into a fixed-size array")?; - Ok(u64::from_be_bytes(*fixed_arr)) + let vfs = { + // align the name of the directory with the root dir + vfs_dir.name = root_path.file_name().unwrap().to_string_lossy().to_string(); + + let fs_root = VfsRoot { + dir: vfs_dir, + root_path: root_path.clone(), + start_file_offset: 0, + }; + Arc::new(FileBackedVfs::new(Cow::Borrowed(vfs_files_data), fs_root)) + }; + let fs: Arc = + Arc::new(DenoCompileFileSystem::new(vfs.clone())); + Ok(Some(StandaloneData { + fs, + metadata, + modules: StandaloneModules { + remote_modules, + vfs: vfs.clone(), + }, + npm_snapshot, + root_path, + vfs, + })) } pub struct DenoCompileBinaryWriter<'a> { + cjs_tracker: &'a CjsTracker, + cli_options: &'a CliOptions, deno_dir: &'a DenoDir, + emitter: &'a Emitter, file_fetcher: &'a FileFetcher, http_client_provider: &'a HttpClientProvider, npm_resolver: &'a dyn CliNpmResolver, @@ -342,7 +382,10 @@ pub struct DenoCompileBinaryWriter<'a> { impl<'a> DenoCompileBinaryWriter<'a> { #[allow(clippy::too_many_arguments)] pub fn new( + cjs_tracker: &'a CjsTracker, + cli_options: &'a CliOptions, deno_dir: &'a DenoDir, + emitter: &'a Emitter, file_fetcher: &'a FileFetcher, http_client_provider: &'a HttpClientProvider, npm_resolver: &'a dyn CliNpmResolver, @@ -350,7 +393,10 @@ impl<'a> DenoCompileBinaryWriter<'a> { npm_system_info: NpmSystemInfo, ) -> Self { Self { + cjs_tracker, + cli_options, deno_dir, + emitter, file_fetcher, http_client_provider, npm_resolver, @@ -362,11 +408,11 @@ impl<'a> DenoCompileBinaryWriter<'a> { pub async fn write_bin( &self, writer: File, - eszip: eszip::EszipV2, - root_dir_url: EszipRelativeFileBaseUrl<'_>, + graph: &ModuleGraph, + root_dir_url: StandaloneRelativeFileBaseUrl<'_>, entrypoint: &ModuleSpecifier, + include_files: &[ModuleSpecifier], compile_flags: &CompileFlags, - cli_options: &CliOptions, ) -> Result<(), AnyError> { // Select base binary based on target let mut original_binary = self.get_base_binary(compile_flags).await?; @@ -379,7 +425,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { target, ) } - set_windows_binary_to_gui(&mut original_binary)?; + set_windows_binary_to_gui(&mut original_binary) + .context("Setting windows binary to GUI.")?; } if compile_flags.icon.is_some() { let target = compile_flags.resolve_target(); @@ -390,15 +437,17 @@ impl<'a> DenoCompileBinaryWriter<'a> { ) } } - self.write_standalone_binary( - writer, - original_binary, - eszip, - root_dir_url, - entrypoint, - cli_options, - compile_flags, - ) + self + .write_standalone_binary( + writer, + original_binary, + graph, + root_dir_url, + entrypoint, + include_files, + compile_flags, + ) + .await } async fn get_base_binary( @@ -409,7 +458,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { // // Phase 2 of the 'min sized' deno compile RFC talks // about adding this as a flag. - if let Some(path) = std::env::var_os("DENORT_BIN") { + if let Some(path) = get_dev_binary_path() { return std::fs::read(&path).with_context(|| { format!("Could not find denort at '{}'", path.to_string_lossy()) }); @@ -427,13 +476,9 @@ impl<'a> DenoCompileBinaryWriter<'a> { binary_name ) } - ReleaseChannel::Stable => { + _ => { format!("release/v{}/{}", env!("CARGO_PKG_VERSION"), binary_name) } - _ => bail!( - "`deno compile` current doesn't support {} release channel", - crate::version::DENO_VERSION_INFO.release_channel.name() - ), }; let download_directory = self.deno_dir.dl_folder_path(); @@ -442,10 +487,14 @@ impl<'a> DenoCompileBinaryWriter<'a> { if !binary_path.exists() { self .download_base_binary(&download_directory, &binary_path_suffix) - .await?; + .await + .context("Setting up base binary.")?; } - let archive_data = std::fs::read(binary_path)?; + let read_file = |path: &Path| -> Result, AnyError> { + std::fs::read(path).with_context(|| format!("Reading {}", path.display())) + }; + let archive_data = read_file(&binary_path)?; let temp_dir = tempfile::TempDir::new()?; let base_binary_path = archive::unpack_into_dir(archive::UnpackArgs { exe_name: "denort", @@ -454,7 +503,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { is_windows: target.contains("windows"), dest_path: temp_dir.path(), })?; - let base_binary = std::fs::read(base_binary_path)?; + let base_binary = read_file(&base_binary_path)?; drop(temp_dir); // delete the temp dir Ok(base_binary) } @@ -472,117 +521,218 @@ impl<'a> DenoCompileBinaryWriter<'a> { self .http_client_provider .get_or_create()? - .download_with_progress(download_url.parse()?, None, &progress) + .download_with_progress_and_retries( + download_url.parse()?, + None, + &progress, + ) .await? }; let bytes = match maybe_bytes { Some(bytes) => bytes, None => { - log::info!("Download could not be found, aborting"); - std::process::exit(1) + bail!("Download could not be found, aborting"); } }; - std::fs::create_dir_all(output_directory)?; + let create_dir_all = |dir: &Path| { + std::fs::create_dir_all(dir) + .with_context(|| format!("Creating {}", dir.display())) + }; + create_dir_all(output_directory)?; let output_path = output_directory.join(binary_path_suffix); - std::fs::create_dir_all(output_path.parent().unwrap())?; - tokio::fs::write(output_path, bytes).await?; + create_dir_all(output_path.parent().unwrap())?; + std::fs::write(&output_path, bytes) + .with_context(|| format!("Writing {}", output_path.display()))?; Ok(()) } /// This functions creates a standalone deno binary by appending a bundle /// and magic trailer to the currently executing binary. #[allow(clippy::too_many_arguments)] - fn write_standalone_binary( + async fn write_standalone_binary( &self, writer: File, original_bin: Vec, - mut eszip: eszip::EszipV2, - root_dir_url: EszipRelativeFileBaseUrl<'_>, + graph: &ModuleGraph, + root_dir_url: StandaloneRelativeFileBaseUrl<'_>, entrypoint: &ModuleSpecifier, - cli_options: &CliOptions, + include_files: &[ModuleSpecifier], compile_flags: &CompileFlags, ) -> Result<(), AnyError> { - let ca_data = match cli_options.ca_data() { + let ca_data = match self.cli_options.ca_data() { Some(CaData::File(ca_file)) => Some( - std::fs::read(ca_file) - .with_context(|| format!("Reading: {ca_file}"))?, + std::fs::read(ca_file).with_context(|| format!("Reading {ca_file}"))?, ), Some(CaData::Bytes(bytes)) => Some(bytes.clone()), None => None, }; let root_path = root_dir_url.inner().to_file_path().unwrap(); - let (npm_vfs, npm_files, node_modules) = match self.npm_resolver.as_inner() - { - InnerCliNpmResolverRef::Managed(managed) => { - let snapshot = - managed.serialized_valid_snapshot_for_system(&self.npm_system_info); - if !snapshot.as_serialized().packages.is_empty() { - let (root_dir, files) = self - .build_vfs(&root_path, cli_options)? - .into_dir_and_files(); - eszip.add_npm_snapshot(snapshot); + let (maybe_npm_vfs, node_modules, npm_snapshot) = + match self.npm_resolver.as_inner() { + InnerCliNpmResolverRef::Managed(managed) => { + let snapshot = + managed.serialized_valid_snapshot_for_system(&self.npm_system_info); + if !snapshot.as_serialized().packages.is_empty() { + let npm_vfs_builder = self + .build_npm_vfs(&root_path) + .context("Building npm vfs.")?; + ( + Some(npm_vfs_builder), + Some(NodeModules::Managed { + node_modules_dir: self + .npm_resolver + .root_node_modules_path() + .map(|path| { + root_dir_url + .specifier_key( + &ModuleSpecifier::from_directory_path(path).unwrap(), + ) + .into_owned() + }), + }), + Some(snapshot), + ) + } else { + (None, None, None) + } + } + InnerCliNpmResolverRef::Byonm(resolver) => { + let npm_vfs_builder = self.build_npm_vfs(&root_path)?; ( - Some(root_dir), - files, - Some(NodeModules::Managed { - node_modules_dir: self.npm_resolver.root_node_modules_path().map( - |path| { + Some(npm_vfs_builder), + Some(NodeModules::Byonm { + root_node_modules_dir: resolver.root_node_modules_path().map( + |node_modules_dir| { root_dir_url .specifier_key( - &ModuleSpecifier::from_directory_path(path).unwrap(), + &ModuleSpecifier::from_directory_path(node_modules_dir) + .unwrap(), ) .into_owned() }, ), }), + None, ) - } else { - (None, Vec::new(), None) } + }; + let mut vfs = if let Some(npm_vfs) = maybe_npm_vfs { + npm_vfs + } else { + VfsBuilder::new(root_path.clone())? + }; + for include_file in include_files { + let path = deno_path_util::url_to_file_path(include_file)?; + if path.is_dir() { + // TODO(#26941): we should analyze if any of these are + // modules in order to include their dependencies + vfs + .add_dir_recursive(&path) + .with_context(|| format!("Including {}", path.display()))?; + } else { + vfs + .add_file_at_path(&path) + .with_context(|| format!("Including {}", path.display()))?; } - InnerCliNpmResolverRef::Byonm(resolver) => { - let (root_dir, files) = self - .build_vfs(&root_path, cli_options)? - .into_dir_and_files(); - ( - Some(root_dir), - files, - Some(NodeModules::Byonm { - root_node_modules_dir: resolver.root_node_modules_path().map( - |node_modules_dir| { - root_dir_url - .specifier_key( - &ModuleSpecifier::from_directory_path(node_modules_dir) - .unwrap(), - ) - .into_owned() - }, - ), - }), - ) - } + } + let mut remote_modules_store = RemoteModulesStoreBuilder::default(); + let mut code_cache_key_hasher = if self.cli_options.code_cache_enabled() { + Some(FastInsecureHasher::new_deno_versioned()) + } else { + None }; + for module in graph.modules() { + if module.specifier().scheme() == "data" { + continue; // don't store data urls as an entry as they're in the code + } + if let Some(hasher) = &mut code_cache_key_hasher { + if let Some(source) = module.source() { + hasher.write(module.specifier().as_str().as_bytes()); + hasher.write(source.as_bytes()); + } + } + let (maybe_source, media_type) = match module { + deno_graph::Module::Js(m) => { + let source = if m.media_type.is_emittable() { + let is_cjs = self.cjs_tracker.is_cjs_with_known_is_script( + &m.specifier, + m.media_type, + m.is_script, + )?; + let module_kind = ModuleKind::from_is_cjs(is_cjs); + let source = self + .emitter + .emit_parsed_source( + &m.specifier, + m.media_type, + module_kind, + &m.source, + ) + .await?; + source.into_bytes() + } else { + m.source.as_bytes().to_vec() + }; + (Some(source), m.media_type) + } + deno_graph::Module::Json(m) => { + (Some(m.source.as_bytes().to_vec()), m.media_type) + } + deno_graph::Module::Wasm(m) => { + (Some(m.source.to_vec()), MediaType::Wasm) + } + deno_graph::Module::Npm(_) + | deno_graph::Module::Node(_) + | deno_graph::Module::External(_) => (None, MediaType::Unknown), + }; + if module.specifier().scheme() == "file" { + let file_path = deno_path_util::url_to_file_path(module.specifier())?; + vfs + .add_file_with_data( + &file_path, + match maybe_source { + Some(source) => source, + None => RealFs.read_file_sync(&file_path, None)?.into_owned(), + }, + VfsFileSubDataKind::ModuleGraph, + ) + .with_context(|| { + format!("Failed adding '{}'", file_path.display()) + })?; + } else if let Some(source) = maybe_source { + remote_modules_store.add(module.specifier(), media_type, source); + } + } + remote_modules_store.add_redirects(&graph.redirects); + + let env_vars_from_env_file = match self.cli_options.env_file_name() { + Some(env_filenames) => { + let mut aggregated_env_vars = IndexMap::new(); + for env_filename in env_filenames.iter().rev() { + log::info!("{} Environment variables from the file \"{}\" were embedded in the generated executable file", crate::colors::yellow("Warning"), env_filename); - let env_vars_from_env_file = match cli_options.env_file_name() { - Some(env_filename) => { - log::info!("{} Environment variables from the file \"{}\" were embedded in the generated executable file", crate::colors::yellow("Warning"), env_filename); - get_file_env_vars(env_filename.to_string())? + let env_vars = get_file_env_vars(env_filename.to_string())?; + aggregated_env_vars.extend(env_vars); + } + aggregated_env_vars } None => Default::default(), }; let metadata = Metadata { argv: compile_flags.args.clone(), - seed: cli_options.seed(), - location: cli_options.location_flag().clone(), - permissions: cli_options.permission_flags().clone(), - v8_flags: cli_options.v8_flags().clone(), - unsafely_ignore_certificate_errors: cli_options + seed: self.cli_options.seed(), + code_cache_key: code_cache_key_hasher.map(|h| h.finish()), + location: self.cli_options.location_flag().clone(), + permissions: self.cli_options.permission_flags().clone(), + v8_flags: self.cli_options.v8_flags().clone(), + unsafely_ignore_certificate_errors: self + .cli_options .unsafely_ignore_certificate_errors() .clone(), - log_level: cli_options.log_level(), - ca_stores: cli_options.ca_stores().clone(), + log_level: self.cli_options.log_level(), + ca_stores: self.cli_options.ca_stores().clone(), ca_data, env_vars_from_env_file, entrypoint_key: root_dir_url.specifier_key(entrypoint).into_owned(), @@ -625,28 +775,27 @@ impl<'a> DenoCompileBinaryWriter<'a> { node_modules, unstable_config: UnstableConfig { legacy_flag_enabled: false, - bare_node_builtins: cli_options.unstable_bare_node_builtins(), - sloppy_imports: cli_options.unstable_sloppy_imports(), - features: cli_options.unstable_features(), + bare_node_builtins: self.cli_options.unstable_bare_node_builtins(), + detect_cjs: self.cli_options.unstable_detect_cjs(), + sloppy_imports: self.cli_options.unstable_sloppy_imports(), + features: self.cli_options.unstable_features(), }, + otel_config: self.cli_options.otel_config(), }; write_binary_bytes( writer, original_bin, &metadata, - eszip, - npm_vfs.as_ref(), - &npm_files, + npm_snapshot.map(|s| s.into_serialized()), + &remote_modules_store, + vfs, compile_flags, ) + .context("Writing binary bytes") } - fn build_vfs( - &self, - root_path: &Path, - cli_options: &CliOptions, - ) -> Result { + fn build_npm_vfs(&self, root_path: &Path) -> Result { fn maybe_warn_different_system(system_info: &NpmSystemInfo) { if system_info != &NpmSystemInfo::default() { log::warn!("{} The node_modules directory may be incompatible with the target system.", crate::colors::yellow("Warning")); @@ -663,8 +812,9 @@ impl<'a> DenoCompileBinaryWriter<'a> { } else { // DO NOT include the user's registry url as it may contain credentials, // but also don't make this dependent on the registry url - let root_path = npm_resolver.global_cache_root_folder(); - let mut builder = VfsBuilder::new(root_path)?; + let global_cache_root_path = npm_resolver.global_cache_root_path(); + let mut builder = + VfsBuilder::new(global_cache_root_path.to_path_buf())?; let mut packages = npm_resolver.all_system_packages(&self.npm_system_info); packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism @@ -674,12 +824,12 @@ impl<'a> DenoCompileBinaryWriter<'a> { builder.add_dir_recursive(&folder)?; } - // Flatten all the registries folders into a single "node_modules/localhost" folder + // Flatten all the registries folders into a single ".deno_compile_node_modules/localhost" folder // that will be used by denort when loading the npm cache. This avoids us exposing // the user's private registry information and means we don't have to bother // serializing all the different registry config into the binary. builder.with_root_dir(|root_dir| { - root_dir.name = "node_modules".to_string(); + root_dir.name = ".deno_compile_node_modules".to_string(); let mut new_entries = Vec::with_capacity(root_dir.entries.len()); let mut localhost_entries = IndexMap::new(); for entry in std::mem::take(&mut root_dir.entries) { @@ -714,19 +864,26 @@ impl<'a> DenoCompileBinaryWriter<'a> { root_dir.entries = new_entries; }); + builder.set_new_root_path(root_path.to_path_buf())?; + Ok(builder) } } InnerCliNpmResolverRef::Byonm(_) => { maybe_warn_different_system(&self.npm_system_info); let mut builder = VfsBuilder::new(root_path.to_path_buf())?; - for pkg_json in cli_options.workspace().package_jsons() { + for pkg_json in self.cli_options.workspace().package_jsons() { builder.add_file_at_path(&pkg_json.path)?; } // traverse and add all the node_modules directories in the workspace let mut pending_dirs = VecDeque::new(); pending_dirs.push_back( - cli_options.workspace().root_dir().to_file_path().unwrap(), + self + .cli_options + .workspace() + .root_dir() + .to_file_path() + .unwrap(), ); while let Some(pending_dir) = pending_dirs.pop_front() { let mut entries = fs::read_dir(&pending_dir) @@ -753,6 +910,31 @@ impl<'a> DenoCompileBinaryWriter<'a> { } } +fn get_denort_path(deno_exe: PathBuf) -> Option { + let mut denort = deno_exe; + denort.set_file_name(if cfg!(windows) { + "denort.exe" + } else { + "denort" + }); + denort.exists().then(|| denort.into_os_string()) +} + +fn get_dev_binary_path() -> Option { + env::var_os("DENORT_BIN").or_else(|| { + env::current_exe().ok().and_then(|exec_path| { + if exec_path + .components() + .any(|component| component == Component::Normal("target".as_ref())) + { + get_denort_path(exec_path) + } else { + None + } + }) + }) +} + /// This function returns the environment variables specified /// in the passed environment file. fn get_file_env_vars( diff --git a/cli/standalone/code_cache.rs b/cli/standalone/code_cache.rs new file mode 100644 index 00000000000000..9580b9b44e16c1 --- /dev/null +++ b/cli/standalone/code_cache.rs @@ -0,0 +1,523 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::io::BufReader; +use std::io::BufWriter; +use std::io::Read; +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use deno_ast::ModuleSpecifier; +use deno_core::anyhow::bail; +use deno_core::error::AnyError; +use deno_core::parking_lot::Mutex; +use deno_core::unsync::sync::AtomicFlag; +use deno_runtime::code_cache::CodeCache; +use deno_runtime::code_cache::CodeCacheType; + +use crate::cache::FastInsecureHasher; +use crate::util::path::get_atomic_file_path; +use crate::worker::CliCodeCache; + +enum CodeCacheStrategy { + FirstRun(FirstRunCodeCacheStrategy), + SubsequentRun(SubsequentRunCodeCacheStrategy), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DenoCompileCodeCacheEntry { + pub source_hash: u64, + pub data: Vec, +} + +pub struct DenoCompileCodeCache { + strategy: CodeCacheStrategy, +} + +impl DenoCompileCodeCache { + pub fn new(file_path: PathBuf, cache_key: u64) -> Self { + // attempt to deserialize the cache data + match deserialize(&file_path, cache_key) { + Ok(data) => { + log::debug!( + "Loaded {} code cache entries from {}", + data.len(), + file_path.display() + ); + Self { + strategy: CodeCacheStrategy::SubsequentRun( + SubsequentRunCodeCacheStrategy { + is_finished: AtomicFlag::lowered(), + data: Mutex::new(data), + }, + ), + } + } + Err(err) => { + log::debug!( + "Failed to deserialize code cache from {}: {:#}", + file_path.display(), + err + ); + Self { + strategy: CodeCacheStrategy::FirstRun(FirstRunCodeCacheStrategy { + cache_key, + file_path, + is_finished: AtomicFlag::lowered(), + data: Mutex::new(FirstRunCodeCacheData { + cache: HashMap::new(), + add_count: 0, + }), + }), + } + } + } + } +} + +impl CodeCache for DenoCompileCodeCache { + fn get_sync( + &self, + specifier: &ModuleSpecifier, + code_cache_type: CodeCacheType, + source_hash: u64, + ) -> Option> { + match &self.strategy { + CodeCacheStrategy::FirstRun(strategy) => { + if !strategy.is_finished.is_raised() { + // we keep track of how many times the cache is requested + // then serialize the cache when we get that number of + // "set" calls + strategy.data.lock().add_count += 1; + } + None + } + CodeCacheStrategy::SubsequentRun(strategy) => { + if strategy.is_finished.is_raised() { + return None; + } + strategy.take_from_cache(specifier, code_cache_type, source_hash) + } + } + } + + fn set_sync( + &self, + specifier: ModuleSpecifier, + code_cache_type: CodeCacheType, + source_hash: u64, + bytes: &[u8], + ) { + match &self.strategy { + CodeCacheStrategy::FirstRun(strategy) => { + if strategy.is_finished.is_raised() { + return; + } + + let data_to_serialize = { + let mut data = strategy.data.lock(); + data.cache.insert( + (specifier.to_string(), code_cache_type), + DenoCompileCodeCacheEntry { + source_hash, + data: bytes.to_vec(), + }, + ); + if data.add_count != 0 { + data.add_count -= 1; + } + if data.add_count == 0 { + // don't allow using the cache anymore + strategy.is_finished.raise(); + if data.cache.is_empty() { + None + } else { + Some(std::mem::take(&mut data.cache)) + } + } else { + None + } + }; + if let Some(cache_data) = &data_to_serialize { + strategy.write_cache_data(cache_data); + } + } + CodeCacheStrategy::SubsequentRun(_) => { + // do nothing + } + } + } +} + +impl CliCodeCache for DenoCompileCodeCache { + fn enabled(&self) -> bool { + match &self.strategy { + CodeCacheStrategy::FirstRun(strategy) => { + !strategy.is_finished.is_raised() + } + CodeCacheStrategy::SubsequentRun(strategy) => { + !strategy.is_finished.is_raised() + } + } + } + + fn as_code_cache(self: Arc) -> Arc { + self + } +} + +type CodeCacheKey = (String, CodeCacheType); + +struct FirstRunCodeCacheData { + cache: HashMap, + add_count: usize, +} + +struct FirstRunCodeCacheStrategy { + cache_key: u64, + file_path: PathBuf, + is_finished: AtomicFlag, + data: Mutex, +} + +impl FirstRunCodeCacheStrategy { + fn write_cache_data( + &self, + cache_data: &HashMap, + ) { + let count = cache_data.len(); + let temp_file = get_atomic_file_path(&self.file_path); + match serialize(&temp_file, self.cache_key, cache_data) { + Ok(()) => { + if let Err(err) = std::fs::rename(&temp_file, &self.file_path) { + log::debug!("Failed to rename code cache: {}", err); + let _ = std::fs::remove_file(&temp_file); + } else { + log::debug!("Serialized {} code cache entries", count); + } + } + Err(err) => { + let _ = std::fs::remove_file(&temp_file); + log::debug!("Failed to serialize code cache: {}", err); + } + } + } +} + +struct SubsequentRunCodeCacheStrategy { + is_finished: AtomicFlag, + data: Mutex>, +} + +impl SubsequentRunCodeCacheStrategy { + fn take_from_cache( + &self, + specifier: &ModuleSpecifier, + code_cache_type: CodeCacheType, + source_hash: u64, + ) -> Option> { + let mut data = self.data.lock(); + // todo(dsherret): how to avoid the clone here? + let entry = data.remove(&(specifier.to_string(), code_cache_type))?; + if entry.source_hash != source_hash { + return None; + } + if data.is_empty() { + self.is_finished.raise(); + } + Some(entry.data) + } +} + +/// File format: +/// -
+/// - +/// - +/// - <[entry length]> - u64 * number of entries +/// - <[entry]> +/// - <[u8]: entry data> +/// - +/// - : code cache type +/// - +/// - +/// - +fn serialize( + file_path: &Path, + cache_key: u64, + cache: &HashMap, +) -> Result<(), AnyError> { + let cache_file = std::fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(file_path)?; + let mut writer = BufWriter::new(cache_file); + serialize_with_writer(&mut writer, cache_key, cache) +} + +fn serialize_with_writer( + writer: &mut BufWriter, + cache_key: u64, + cache: &HashMap, +) -> Result<(), AnyError> { + // header + writer.write_all(&cache_key.to_le_bytes())?; + writer.write_all(&(cache.len() as u32).to_le_bytes())?; + // lengths of each entry + for ((specifier, _), entry) in cache { + let len: u64 = + entry.data.len() as u64 + specifier.len() as u64 + 1 + 4 + 8 + 8; + writer.write_all(&len.to_le_bytes())?; + } + // entries + for ((specifier, code_cache_type), entry) in cache { + writer.write_all(&entry.data)?; + writer.write_all(&[match code_cache_type { + CodeCacheType::EsModule => 0, + CodeCacheType::Script => 1, + }])?; + writer.write_all(specifier.as_bytes())?; + writer.write_all(&(specifier.len() as u32).to_le_bytes())?; + writer.write_all(&entry.source_hash.to_le_bytes())?; + let hash: u64 = FastInsecureHasher::new_without_deno_version() + .write(&entry.data) + .finish(); + writer.write_all(&hash.to_le_bytes())?; + } + + writer.flush()?; + + Ok(()) +} + +fn deserialize( + file_path: &Path, + expected_cache_key: u64, +) -> Result, AnyError> { + let cache_file = std::fs::File::open(file_path)?; + let mut reader = BufReader::new(cache_file); + deserialize_with_reader(&mut reader, expected_cache_key) +} + +fn deserialize_with_reader( + reader: &mut BufReader, + expected_cache_key: u64, +) -> Result, AnyError> { + // it's very important to use this below so that a corrupt cache file + // doesn't cause a memory allocation error + fn new_vec_sized( + capacity: usize, + default_value: T, + ) -> Result, AnyError> { + let mut vec = Vec::new(); + vec.try_reserve(capacity)?; + vec.resize(capacity, default_value); + Ok(vec) + } + + fn try_subtract(a: usize, b: usize) -> Result { + if a < b { + bail!("Integer underflow"); + } + Ok(a - b) + } + + let mut header_bytes = vec![0; 8 + 4]; + reader.read_exact(&mut header_bytes)?; + let actual_cache_key = u64::from_le_bytes(header_bytes[..8].try_into()?); + if actual_cache_key != expected_cache_key { + // cache bust + bail!("Cache key mismatch"); + } + let len = u32::from_le_bytes(header_bytes[8..].try_into()?) as usize; + // read the lengths for each entry found in the file + let entry_len_bytes_capacity = len * 8; + let mut entry_len_bytes = new_vec_sized(entry_len_bytes_capacity, 0)?; + reader.read_exact(&mut entry_len_bytes)?; + let mut lengths = Vec::new(); + lengths.try_reserve(len)?; + for i in 0..len { + let pos = i * 8; + lengths.push( + u64::from_le_bytes(entry_len_bytes[pos..pos + 8].try_into()?) as usize, + ); + } + + let mut map = HashMap::new(); + map.try_reserve(len)?; + for len in lengths { + let mut buffer = new_vec_sized(len, 0)?; + reader.read_exact(&mut buffer)?; + let entry_data_hash_start_pos = try_subtract(buffer.len(), 8)?; + let expected_entry_data_hash = + u64::from_le_bytes(buffer[entry_data_hash_start_pos..].try_into()?); + let source_hash_start_pos = try_subtract(entry_data_hash_start_pos, 8)?; + let source_hash = u64::from_le_bytes( + buffer[source_hash_start_pos..entry_data_hash_start_pos].try_into()?, + ); + let specifier_end_pos = try_subtract(source_hash_start_pos, 4)?; + let specifier_len = u32::from_le_bytes( + buffer[specifier_end_pos..source_hash_start_pos].try_into()?, + ) as usize; + let specifier_start_pos = try_subtract(specifier_end_pos, specifier_len)?; + let specifier = String::from_utf8( + buffer[specifier_start_pos..specifier_end_pos].to_vec(), + )?; + let code_cache_type_pos = try_subtract(specifier_start_pos, 1)?; + let code_cache_type = match buffer[code_cache_type_pos] { + 0 => CodeCacheType::EsModule, + 1 => CodeCacheType::Script, + _ => bail!("Invalid code cache type"), + }; + buffer.truncate(code_cache_type_pos); + let actual_entry_data_hash: u64 = + FastInsecureHasher::new_without_deno_version() + .write(&buffer) + .finish(); + if expected_entry_data_hash != actual_entry_data_hash { + bail!("Hash mismatch.") + } + map.insert( + (specifier, code_cache_type), + DenoCompileCodeCacheEntry { + source_hash, + data: buffer, + }, + ); + } + + Ok(map) +} + +#[cfg(test)] +mod test { + use test_util::TempDir; + + use super::*; + use std::fs::File; + + #[test] + fn serialize_deserialize() { + let cache_key = 123456; + let cache = { + let mut cache = HashMap::new(); + cache.insert( + ("specifier1".to_string(), CodeCacheType::EsModule), + DenoCompileCodeCacheEntry { + source_hash: 1, + data: vec![1, 2, 3], + }, + ); + cache.insert( + ("specifier2".to_string(), CodeCacheType::EsModule), + DenoCompileCodeCacheEntry { + source_hash: 2, + data: vec![4, 5, 6], + }, + ); + cache.insert( + ("specifier2".to_string(), CodeCacheType::Script), + DenoCompileCodeCacheEntry { + source_hash: 2, + data: vec![6, 5, 1], + }, + ); + cache + }; + let mut buffer = Vec::new(); + serialize_with_writer(&mut BufWriter::new(&mut buffer), cache_key, &cache) + .unwrap(); + let deserialized = + deserialize_with_reader(&mut BufReader::new(&buffer[..]), cache_key) + .unwrap(); + assert_eq!(cache, deserialized); + } + + #[test] + fn serialize_deserialize_empty() { + let cache_key = 1234; + let cache = HashMap::new(); + let mut buffer = Vec::new(); + serialize_with_writer(&mut BufWriter::new(&mut buffer), cache_key, &cache) + .unwrap(); + let deserialized = + deserialize_with_reader(&mut BufReader::new(&buffer[..]), cache_key) + .unwrap(); + assert_eq!(cache, deserialized); + } + + #[test] + fn serialize_deserialize_corrupt() { + let buffer = "corrupttestingtestingtesting".as_bytes().to_vec(); + let err = deserialize_with_reader(&mut BufReader::new(&buffer[..]), 1234) + .unwrap_err(); + assert_eq!(err.to_string(), "Cache key mismatch"); + } + + #[test] + fn code_cache() { + let temp_dir = TempDir::new(); + let file_path = temp_dir.path().join("cache.bin").to_path_buf(); + let url1 = ModuleSpecifier::parse("https://deno.land/example1.js").unwrap(); + let url2 = ModuleSpecifier::parse("https://deno.land/example2.js").unwrap(); + // first run + { + let code_cache = DenoCompileCodeCache::new(file_path.clone(), 1234); + assert!(code_cache + .get_sync(&url1, CodeCacheType::EsModule, 0) + .is_none()); + assert!(code_cache + .get_sync(&url2, CodeCacheType::EsModule, 1) + .is_none()); + assert!(code_cache.enabled()); + code_cache.set_sync(url1.clone(), CodeCacheType::EsModule, 0, &[1, 2, 3]); + assert!(code_cache.enabled()); + assert!(!file_path.exists()); + code_cache.set_sync(url2.clone(), CodeCacheType::EsModule, 1, &[2, 1, 3]); + assert!(file_path.exists()); // now the new code cache exists + assert!(!code_cache.enabled()); // no longer enabled + } + // second run + { + let code_cache = DenoCompileCodeCache::new(file_path.clone(), 1234); + assert!(code_cache.enabled()); + let result1 = code_cache + .get_sync(&url1, CodeCacheType::EsModule, 0) + .unwrap(); + assert!(code_cache.enabled()); + let result2 = code_cache + .get_sync(&url2, CodeCacheType::EsModule, 1) + .unwrap(); + assert!(!code_cache.enabled()); // no longer enabled + assert_eq!(result1, vec![1, 2, 3]); + assert_eq!(result2, vec![2, 1, 3]); + } + + // new cache key first run + { + let code_cache = DenoCompileCodeCache::new(file_path.clone(), 54321); + assert!(code_cache + .get_sync(&url1, CodeCacheType::EsModule, 0) + .is_none()); + assert!(code_cache + .get_sync(&url2, CodeCacheType::EsModule, 1) + .is_none()); + code_cache.set_sync(url1.clone(), CodeCacheType::EsModule, 0, &[2, 2, 3]); + code_cache.set_sync(url2.clone(), CodeCacheType::EsModule, 1, &[3, 2, 3]); + } + // new cache key second run + { + let code_cache = DenoCompileCodeCache::new(file_path.clone(), 54321); + let result1 = code_cache + .get_sync(&url1, CodeCacheType::EsModule, 0) + .unwrap(); + assert_eq!(result1, vec![2, 2, 3]); + assert!(code_cache + .get_sync(&url2, CodeCacheType::EsModule, 5) // different hash will cause none + .is_none()); + } + } +} diff --git a/cli/standalone/file_system.rs b/cli/standalone/file_system.rs index 536b17f2771374..48dc907570babd 100644 --- a/cli/standalone/file_system.rs +++ b/cli/standalone/file_system.rs @@ -17,13 +17,14 @@ use deno_runtime::deno_io::fs::FsResult; use deno_runtime::deno_io::fs::FsStat; use super::virtual_fs::FileBackedVfs; +use super::virtual_fs::VfsFileSubDataKind; #[derive(Debug, Clone)] pub struct DenoCompileFileSystem(Arc); impl DenoCompileFileSystem { - pub fn new(vfs: FileBackedVfs) -> Self { - Self(Arc::new(vfs)) + pub fn new(vfs: Arc) -> Self { + Self(vfs) } fn error_if_in_vfs(&self, path: &Path) -> FsResult<()> { @@ -36,7 +37,8 @@ impl DenoCompileFileSystem { fn copy_to_real_path(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> { let old_file = self.0.file_entry(oldpath)?; - let old_file_bytes = self.0.read_file_all(old_file)?; + let old_file_bytes = + self.0.read_file_all(old_file, VfsFileSubDataKind::Raw)?; RealFs.write_file_sync( newpath, OpenOptions { @@ -102,7 +104,7 @@ impl FileSystem for DenoCompileFileSystem { &self, path: &Path, recursive: bool, - mode: u32, + mode: Option, ) -> FsResult<()> { self.error_if_in_vfs(path)?; RealFs.mkdir_sync(path, recursive, mode) @@ -111,7 +113,7 @@ impl FileSystem for DenoCompileFileSystem { &self, path: PathBuf, recursive: bool, - mode: u32, + mode: Option, ) -> FsResult<()> { self.error_if_in_vfs(&path)?; RealFs.mkdir_async(path, recursive, mode).await diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index bab26673479167..53efab2964da46 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -5,7 +5,11 @@ #![allow(dead_code)] #![allow(unused_imports)] +use binary::StandaloneData; +use binary::StandaloneModules; +use code_cache::DenoCompileCodeCache; use deno_ast::MediaType; +use deno_cache_dir::npm::NpmCacheDir; use deno_config::workspace::MappedResolution; use deno_config::workspace::MappedResolutionError; use deno_config::workspace::ResolverWorkspaceJsrPackage; @@ -14,8 +18,10 @@ use deno_core::anyhow::Context; use deno_core::error::generic_error; use deno_core::error::type_error; use deno_core::error::AnyError; +use deno_core::futures::future::LocalBoxFuture; use deno_core::futures::FutureExt; use deno_core::v8_set_flags; +use deno_core::FastString; use deno_core::FeatureChecker; use deno_core::ModuleLoader; use deno_core::ModuleSourceCode; @@ -23,11 +29,16 @@ use deno_core::ModuleSpecifier; use deno_core::ModuleType; use deno_core::RequestedModuleType; use deno_core::ResolutionKind; +use deno_core::SourceCodeCacheInfo; use deno_npm::npm_rc::ResolvedNpmRc; use deno_package_json::PackageJsonDepValue; +use deno_resolver::cjs::IsCjsResolutionMode; +use deno_resolver::npm::NpmReqResolverOptions; use deno_runtime::deno_fs; use deno_runtime::deno_node::create_host_defined_options; +use deno_runtime::deno_node::NodeRequireLoader; use deno_runtime::deno_node::NodeResolver; +use deno_runtime::deno_node::PackageJsonResolver; use deno_runtime::deno_permissions::Permissions; use deno_runtime::deno_permissions::PermissionsContainer; use deno_runtime::deno_tls::rustls::RootCertStore; @@ -37,13 +48,17 @@ use deno_runtime::permissions::RuntimePermissionDescriptorParser; use deno_runtime::WorkerExecutionMode; use deno_runtime::WorkerLogLevel; use deno_semver::npm::NpmPackageReqReference; -use eszip::EszipRelativeFileBaseUrl; use import_map::parse_from_json; use node_resolver::analyze::NodeCodeTranslator; -use node_resolver::NodeResolutionMode; +use node_resolver::errors::ClosestPkgJsonError; +use node_resolver::NodeResolutionKind; +use node_resolver::ResolutionMode; +use serialization::DenoCompileModuleSource; use std::borrow::Cow; use std::rc::Rc; use std::sync::Arc; +use virtual_fs::FileBackedVfs; +use virtual_fs::VfsFileSubDataKind; use crate::args::create_default_npmrc; use crate::args::get_root_cert_store; @@ -53,85 +68,101 @@ use crate::args::CacheSetting; use crate::args::NpmInstallDepsProvider; use crate::args::StorageKeyResolver; use crate::cache::Caches; +use crate::cache::DenoCacheEnvFsAdapter; use crate::cache::DenoDirProvider; +use crate::cache::FastInsecureHasher; use crate::cache::NodeAnalysisCache; +use crate::cache::RealDenoCacheEnv; use crate::http_util::HttpClientProvider; use crate::node::CliCjsCodeAnalyzer; +use crate::node::CliNodeCodeTranslator; use crate::npm::create_cli_npm_resolver; -use crate::npm::CliNpmResolverByonmCreateOptions; +use crate::npm::create_in_npm_pkg_checker; +use crate::npm::CliByonmNpmResolverCreateOptions; +use crate::npm::CliManagedInNpmPkgCheckerCreateOptions; +use crate::npm::CliManagedNpmResolverCreateOptions; +use crate::npm::CliNpmResolver; use crate::npm::CliNpmResolverCreateOptions; -use crate::npm::CliNpmResolverManagedCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; -use crate::npm::NpmCacheDir; -use crate::resolver::CjsResolutionStore; -use crate::resolver::CliNodeResolver; +use crate::npm::CreateInNpmPkgCheckerOptions; +use crate::resolver::CjsTracker; +use crate::resolver::CliDenoResolverFs; +use crate::resolver::CliNpmReqResolver; use crate::resolver::NpmModuleLoader; use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; +use crate::util::text_encoding::from_utf8_lossy_cow; use crate::util::v8::construct_v8_flags; +use crate::worker::CliCodeCache; use crate::worker::CliMainWorkerFactory; use crate::worker::CliMainWorkerOptions; -use crate::worker::ModuleLoaderAndSourceMapGetter; +use crate::worker::CreateModuleLoaderResult; use crate::worker::ModuleLoaderFactory; pub mod binary; +mod code_cache; mod file_system; +mod serialization; mod virtual_fs; pub use binary::extract_standalone; pub use binary::is_standalone_binary; pub use binary::DenoCompileBinaryWriter; -use self::binary::load_npm_vfs; use self::binary::Metadata; use self::file_system::DenoCompileFileSystem; -struct WorkspaceEszipModule { - specifier: ModuleSpecifier, - inner: eszip::Module, -} - -struct WorkspaceEszip { - eszip: eszip::EszipV2, - root_dir_url: Arc, +struct SharedModuleLoaderState { + cjs_tracker: Arc, + code_cache: Option>, + fs: Arc, + modules: StandaloneModules, + node_code_translator: Arc, + node_resolver: Arc, + npm_module_loader: Arc, + npm_req_resolver: Arc, + npm_resolver: Arc, + vfs: Arc, + workspace_resolver: WorkspaceResolver, } -impl WorkspaceEszip { - pub fn get_module( +impl SharedModuleLoaderState { + fn get_code_cache( &self, specifier: &ModuleSpecifier, - ) -> Option { - if specifier.scheme() == "file" { - let specifier_key = EszipRelativeFileBaseUrl::new(&self.root_dir_url) - .specifier_key(specifier); - let module = self.eszip.get_module(&specifier_key)?; - let specifier = self.root_dir_url.join(&module.specifier).unwrap(); - Some(WorkspaceEszipModule { - specifier, - inner: module, - }) - } else { - let module = self.eszip.get_module(specifier.as_str())?; - Some(WorkspaceEszipModule { - specifier: ModuleSpecifier::parse(&module.specifier).unwrap(), - inner: module, - }) + source: &[u8], + ) -> Option { + let Some(code_cache) = &self.code_cache else { + return None; + }; + if !code_cache.enabled() { + return None; } + // deno version is already included in the root cache key + let hash = FastInsecureHasher::new_without_deno_version() + .write_hashable(source) + .finish(); + let data = code_cache.get_sync( + specifier, + deno_runtime::code_cache::CodeCacheType::EsModule, + hash, + ); + Some(SourceCodeCacheInfo { + hash, + data: data.map(Cow::Owned), + }) } } -struct SharedModuleLoaderState { - eszip: WorkspaceEszip, - workspace_resolver: WorkspaceResolver, - node_resolver: Arc, - npm_module_loader: Arc, -} - #[derive(Clone)] struct EmbeddedModuleLoader { shared: Arc, - root_permissions: PermissionsContainer, - dynamic_permissions: PermissionsContainer, +} + +impl std::fmt::Debug for EmbeddedModuleLoader { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("EmbeddedModuleLoader").finish() + } } pub const MODULE_NOT_FOUND: &str = "Module not found"; @@ -158,13 +189,27 @@ impl ModuleLoader for EmbeddedModuleLoader { type_error(format!("Referrer uses invalid specifier: {}", err)) })? }; + let referrer_kind = if self + .shared + .cjs_tracker + .is_maybe_cjs(&referrer, MediaType::from_specifier(&referrer))? + { + ResolutionMode::Require + } else { + ResolutionMode::Import + }; if self.shared.node_resolver.in_npm_package(&referrer) { return Ok( self .shared .node_resolver - .resolve(raw_specifier, &referrer, NodeResolutionMode::Execution)? + .resolve( + raw_specifier, + &referrer, + referrer_kind, + NodeResolutionKind::Execution, + )? .into_url(), ); } @@ -186,13 +231,13 @@ impl ModuleLoader for EmbeddedModuleLoader { self .shared .node_resolver - .resolve_package_sub_path_from_deno_module( + .resolve_package_subpath_from_deno_module( pkg_json.dir_path(), sub_path.as_deref(), Some(&referrer), - NodeResolutionMode::Execution, - )? - .into_url(), + referrer_kind, + NodeResolutionKind::Execution, + )?, ), Ok(MappedResolution::PackageJson { dep_result, @@ -202,14 +247,15 @@ impl ModuleLoader for EmbeddedModuleLoader { }) => match dep_result.as_ref().map_err(|e| AnyError::from(e.clone()))? { PackageJsonDepValue::Req(req) => self .shared - .node_resolver + .npm_req_resolver .resolve_req_with_sub_path( req, sub_path.as_deref(), &referrer, - NodeResolutionMode::Execution, + referrer_kind, + NodeResolutionKind::Execution, ) - .map(|res| res.into_url()), + .map_err(AnyError::from), PackageJsonDepValue::Workspace(version_req) => { let pkg_folder = self .shared @@ -222,13 +268,13 @@ impl ModuleLoader for EmbeddedModuleLoader { self .shared .node_resolver - .resolve_package_sub_path_from_deno_module( + .resolve_package_subpath_from_deno_module( pkg_folder, sub_path.as_deref(), Some(&referrer), - NodeResolutionMode::Execution, - )? - .into_url(), + referrer_kind, + NodeResolutionKind::Execution, + )?, ) } }, @@ -237,20 +283,19 @@ impl ModuleLoader for EmbeddedModuleLoader { if let Ok(reference) = NpmPackageReqReference::from_specifier(&specifier) { - return self - .shared - .node_resolver - .resolve_req_reference( - &reference, - &referrer, - NodeResolutionMode::Execution, - ) - .map(|res| res.into_url()); + return Ok(self.shared.npm_req_resolver.resolve_req_reference( + &reference, + &referrer, + referrer_kind, + NodeResolutionKind::Execution, + )?); } if specifier.scheme() == "jsr" { - if let Some(module) = self.shared.eszip.get_module(&specifier) { - return Ok(module.specifier); + if let Some(specifier) = + self.shared.modules.resolve_specifier(&specifier)? + { + return Ok(specifier.clone()); } } @@ -258,17 +303,18 @@ impl ModuleLoader for EmbeddedModuleLoader { self .shared .node_resolver - .handle_if_in_node_modules(&specifier)? + .handle_if_in_node_modules(&specifier) .unwrap_or(specifier), ) } Err(err) if err.is_unmapped_bare_specifier() && referrer.scheme() == "file" => { - let maybe_res = self.shared.node_resolver.resolve_if_for_npm_pkg( + let maybe_res = self.shared.npm_req_resolver.resolve_if_for_npm_pkg( raw_specifier, &referrer, - NodeResolutionMode::Execution, + referrer_kind, + NodeResolutionKind::Execution, )?; if let Some(res) = maybe_res { return Ok(res.into_url()); @@ -322,14 +368,19 @@ impl ModuleLoader for EmbeddedModuleLoader { } if self.shared.node_resolver.in_npm_package(original_specifier) { - let npm_module_loader = self.shared.npm_module_loader.clone(); + let shared = self.shared.clone(); let original_specifier = original_specifier.clone(); let maybe_referrer = maybe_referrer.cloned(); return deno_core::ModuleLoadResponse::Async( async move { - let code_source = npm_module_loader + let code_source = shared + .npm_module_loader .load(&original_specifier, maybe_referrer.as_ref()) .await?; + let code_cache_entry = shared.get_code_cache( + &code_source.found_url, + code_source.code.as_bytes(), + ); Ok(deno_core::ModuleSource::new_with_redirect( match code_source.media_type { MediaType::Json => ModuleType::Json, @@ -338,94 +389,182 @@ impl ModuleLoader for EmbeddedModuleLoader { code_source.code, &original_specifier, &code_source.found_url, - None, + code_cache_entry, )) } .boxed_local(), ); } - let Some(module) = self.shared.eszip.get_module(original_specifier) else { - return deno_core::ModuleLoadResponse::Sync(Err(type_error(format!( - "{MODULE_NOT_FOUND}: {}", - original_specifier - )))); - }; - let original_specifier = original_specifier.clone(); - - deno_core::ModuleLoadResponse::Async( - async move { - let code = module.inner.source().await.ok_or_else(|| { - type_error(format!("Module not found: {}", original_specifier)) - })?; - let code = arc_u8_to_arc_str(code) - .map_err(|_| type_error("Module source is not utf-8"))?; - Ok(deno_core::ModuleSource::new_with_redirect( - match module.inner.kind { - eszip::ModuleKind::JavaScript => ModuleType::JavaScript, - eszip::ModuleKind::Json => ModuleType::Json, - eszip::ModuleKind::Jsonc => { - return Err(type_error("jsonc modules not supported")) - } - eszip::ModuleKind::OpaqueData => { - unreachable!(); + match self.shared.modules.read(original_specifier) { + Ok(Some(module)) => { + let media_type = module.media_type; + let (module_specifier, module_type, module_source) = + module.into_parts(); + let is_maybe_cjs = match self + .shared + .cjs_tracker + .is_maybe_cjs(original_specifier, media_type) + { + Ok(is_maybe_cjs) => is_maybe_cjs, + Err(err) => { + return deno_core::ModuleLoadResponse::Sync(Err(type_error( + format!("{:?}", err), + ))); + } + }; + if is_maybe_cjs { + let original_specifier = original_specifier.clone(); + let module_specifier = module_specifier.clone(); + let shared = self.shared.clone(); + deno_core::ModuleLoadResponse::Async( + async move { + let source = match module_source { + DenoCompileModuleSource::String(string) => { + Cow::Borrowed(string) + } + DenoCompileModuleSource::Bytes(module_code_bytes) => { + match module_code_bytes { + Cow::Owned(bytes) => Cow::Owned( + crate::util::text_encoding::from_utf8_lossy_owned(bytes), + ), + Cow::Borrowed(bytes) => String::from_utf8_lossy(bytes), + } + } + }; + let source = shared + .node_code_translator + .translate_cjs_to_esm(&module_specifier, Some(source)) + .await?; + let module_source = match source { + Cow::Owned(source) => ModuleSourceCode::String(source.into()), + Cow::Borrowed(source) => { + ModuleSourceCode::String(FastString::from_static(source)) + } + }; + let code_cache_entry = shared + .get_code_cache(&module_specifier, module_source.as_bytes()); + Ok(deno_core::ModuleSource::new_with_redirect( + module_type, + module_source, + &original_specifier, + &module_specifier, + code_cache_entry, + )) } - }, - ModuleSourceCode::String(code.into()), - &original_specifier, - &module.specifier, - None, - )) + .boxed_local(), + ) + } else { + let module_source = module_source.into_for_v8(); + let code_cache_entry = self + .shared + .get_code_cache(module_specifier, module_source.as_bytes()); + deno_core::ModuleLoadResponse::Sync(Ok( + deno_core::ModuleSource::new_with_redirect( + module_type, + module_source, + original_specifier, + module_specifier, + code_cache_entry, + ), + )) + } } - .boxed_local(), - ) + Ok(None) => deno_core::ModuleLoadResponse::Sync(Err(type_error( + format!("{MODULE_NOT_FOUND}: {}", original_specifier), + ))), + Err(err) => deno_core::ModuleLoadResponse::Sync(Err(type_error( + format!("{:?}", err), + ))), + } + } + + fn code_cache_ready( + &self, + specifier: ModuleSpecifier, + source_hash: u64, + code_cache_data: &[u8], + ) -> LocalBoxFuture<'static, ()> { + if let Some(code_cache) = &self.shared.code_cache { + code_cache.set_sync( + specifier, + deno_runtime::code_cache::CodeCacheType::EsModule, + source_hash, + code_cache_data, + ); + } + std::future::ready(()).boxed_local() } } -fn arc_u8_to_arc_str( - arc_u8: Arc<[u8]>, -) -> Result, std::str::Utf8Error> { - // Check that the string is valid UTF-8. - std::str::from_utf8(&arc_u8)?; - // SAFETY: the string is valid UTF-8, and the layout Arc<[u8]> is the same as - // Arc. This is proven by the From> impl for Arc<[u8]> from the - // standard library. - Ok(unsafe { - std::mem::transmute::, std::sync::Arc>(arc_u8) - }) +impl NodeRequireLoader for EmbeddedModuleLoader { + fn ensure_read_permission<'a>( + &self, + permissions: &mut dyn deno_runtime::deno_node::NodePermissions, + path: &'a std::path::Path, + ) -> Result, AnyError> { + if self.shared.modules.has_file(path) { + // allow reading if the file is in the snapshot + return Ok(Cow::Borrowed(path)); + } + + self + .shared + .npm_resolver + .ensure_read_permission(permissions, path) + } + + fn load_text_file_lossy( + &self, + path: &std::path::Path, + ) -> Result, AnyError> { + let file_entry = self.shared.vfs.file_entry(path)?; + let file_bytes = self + .shared + .vfs + .read_file_all(file_entry, VfsFileSubDataKind::ModuleGraph)?; + Ok(from_utf8_lossy_cow(file_bytes)) + } + + fn is_maybe_cjs( + &self, + specifier: &ModuleSpecifier, + ) -> Result { + let media_type = MediaType::from_specifier(specifier); + self.shared.cjs_tracker.is_maybe_cjs(specifier, media_type) + } } struct StandaloneModuleLoaderFactory { shared: Arc, } +impl StandaloneModuleLoaderFactory { + pub fn create_result(&self) -> CreateModuleLoaderResult { + let loader = Rc::new(EmbeddedModuleLoader { + shared: self.shared.clone(), + }); + CreateModuleLoaderResult { + module_loader: loader.clone(), + node_require_loader: loader, + } + } +} + impl ModuleLoaderFactory for StandaloneModuleLoaderFactory { fn create_for_main( &self, - root_permissions: PermissionsContainer, - dynamic_permissions: PermissionsContainer, - ) -> ModuleLoaderAndSourceMapGetter { - ModuleLoaderAndSourceMapGetter { - module_loader: Rc::new(EmbeddedModuleLoader { - shared: self.shared.clone(), - root_permissions, - dynamic_permissions, - }), - } + _root_permissions: PermissionsContainer, + ) -> CreateModuleLoaderResult { + self.create_result() } fn create_for_worker( &self, - root_permissions: PermissionsContainer, - dynamic_permissions: PermissionsContainer, - ) -> ModuleLoaderAndSourceMapGetter { - ModuleLoaderAndSourceMapGetter { - module_loader: Rc::new(EmbeddedModuleLoader { - shared: self.shared.clone(), - root_permissions, - dynamic_permissions, - }), - } + _parent_permissions: PermissionsContainer, + _permissions: PermissionsContainer, + ) -> CreateModuleLoaderResult { + self.create_result() } } @@ -444,13 +583,15 @@ impl RootCertStoreProvider for StandaloneRootCertStoreProvider { } } -pub async fn run( - mut eszip: eszip::EszipV2, - metadata: Metadata, -) -> Result { - let current_exe_path = std::env::current_exe().unwrap(); - let current_exe_name = - current_exe_path.file_name().unwrap().to_string_lossy(); +pub async fn run(data: StandaloneData) -> Result { + let StandaloneData { + fs, + metadata, + modules, + npm_snapshot, + root_path, + vfs, + } = data; let deno_dir_provider = Arc::new(DenoDirProvider::new(None)); let root_cert_store_provider = Arc::new(StandaloneRootCertStoreProvider { ca_stores: metadata.ca_stores, @@ -464,43 +605,50 @@ pub async fn run( )); // use a dummy npm registry url let npm_registry_url = ModuleSpecifier::parse("https://localhost/").unwrap(); - let root_path = - std::env::temp_dir().join(format!("deno-compile-{}", current_exe_name)); let root_dir_url = Arc::new(ModuleSpecifier::from_directory_path(&root_path).unwrap()); let main_module = root_dir_url.join(&metadata.entrypoint_key).unwrap(); - let root_node_modules_path = root_path.join("node_modules"); - let npm_cache_dir = NpmCacheDir::new( - root_node_modules_path.clone(), - vec![npm_registry_url.clone()], - ); - let npm_global_cache_dir = npm_cache_dir.get_cache_location(); + let npm_global_cache_dir = root_path.join(".deno_compile_node_modules"); let cache_setting = CacheSetting::Only; - let (fs, npm_resolver, maybe_vfs_root) = match metadata.node_modules { + let pkg_json_resolver = Arc::new(PackageJsonResolver::new( + deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()), + )); + let (in_npm_pkg_checker, npm_resolver) = match metadata.node_modules { Some(binary::NodeModules::Managed { node_modules_dir }) => { - // this will always have a snapshot - let snapshot = eszip.take_npm_snapshot().unwrap(); - let vfs_root_dir_path = if node_modules_dir.is_some() { - root_path.clone() - } else { - npm_cache_dir.root_dir().to_owned() - }; - let vfs = load_npm_vfs(vfs_root_dir_path.clone()) - .context("Failed to load npm vfs.")?; + // create an npmrc that uses the fake npm_registry_url to resolve packages + let npmrc = Arc::new(ResolvedNpmRc { + default_config: deno_npm::npm_rc::RegistryConfigWithUrl { + registry_url: npm_registry_url.clone(), + config: Default::default(), + }, + scopes: Default::default(), + registry_configs: Default::default(), + }); + let npm_cache_dir = Arc::new(NpmCacheDir::new( + &DenoCacheEnvFsAdapter(fs.as_ref()), + npm_global_cache_dir, + npmrc.get_all_known_registries_urls(), + )); + let snapshot = npm_snapshot.unwrap(); let maybe_node_modules_path = node_modules_dir - .map(|node_modules_dir| vfs_root_dir_path.join(node_modules_dir)); - let fs = Arc::new(DenoCompileFileSystem::new(vfs)) - as Arc; + .map(|node_modules_dir| root_path.join(node_modules_dir)); + let in_npm_pkg_checker = + create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Managed( + CliManagedInNpmPkgCheckerCreateOptions { + root_cache_dir_url: npm_cache_dir.root_dir_url(), + maybe_node_modules_path: maybe_node_modules_path.as_deref(), + }, + )); let npm_resolver = create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed( - CliNpmResolverManagedCreateOptions { + CliManagedNpmResolverCreateOptions { snapshot: CliNpmResolverManagedSnapshotOption::Specified(Some( snapshot, )), maybe_lockfile: None, fs: fs.clone(), http_client_provider: http_client_provider.clone(), - npm_global_cache_dir, + npm_cache_dir, cache_setting, text_only_progress_bar: progress_bar, maybe_node_modules_path, @@ -509,50 +657,54 @@ pub async fn run( // this is only used for installing packages, which isn't necessary with deno compile NpmInstallDepsProvider::empty(), ), - // create an npmrc that uses the fake npm_registry_url to resolve packages - npmrc: Arc::new(ResolvedNpmRc { - default_config: deno_npm::npm_rc::RegistryConfigWithUrl { - registry_url: npm_registry_url.clone(), - config: Default::default(), - }, - scopes: Default::default(), - registry_configs: Default::default(), - }), + npmrc, lifecycle_scripts: Default::default(), }, )) .await?; - (fs, npm_resolver, Some(vfs_root_dir_path)) + (in_npm_pkg_checker, npm_resolver) } Some(binary::NodeModules::Byonm { root_node_modules_dir, }) => { - let vfs_root_dir_path = root_path.clone(); - let vfs = load_npm_vfs(vfs_root_dir_path.clone()) - .context("Failed to load vfs.")?; let root_node_modules_dir = root_node_modules_dir.map(|p| vfs.root().join(p)); - let fs = Arc::new(DenoCompileFileSystem::new(vfs)) - as Arc; + let in_npm_pkg_checker = + create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Byonm); let npm_resolver = create_cli_npm_resolver( - CliNpmResolverCreateOptions::Byonm(CliNpmResolverByonmCreateOptions { - fs: fs.clone(), + CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions { + fs: CliDenoResolverFs(fs.clone()), + pkg_json_resolver: pkg_json_resolver.clone(), root_node_modules_dir, }), ) .await?; - (fs, npm_resolver, Some(vfs_root_dir_path)) + (in_npm_pkg_checker, npm_resolver) } None => { - let fs = Arc::new(deno_fs::RealFs) as Arc; + // Packages from different registries are already inlined in the binary, + // so no need to create actual `.npmrc` configuration. + let npmrc = create_default_npmrc(); + let npm_cache_dir = Arc::new(NpmCacheDir::new( + &DenoCacheEnvFsAdapter(fs.as_ref()), + npm_global_cache_dir, + npmrc.get_all_known_registries_urls(), + )); + let in_npm_pkg_checker = + create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Managed( + CliManagedInNpmPkgCheckerCreateOptions { + root_cache_dir_url: npm_cache_dir.root_dir_url(), + maybe_node_modules_path: None, + }, + )); let npm_resolver = create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed( - CliNpmResolverManagedCreateOptions { + CliManagedNpmResolverCreateOptions { snapshot: CliNpmResolverManagedSnapshotOption::Specified(None), maybe_lockfile: None, fs: fs.clone(), http_client_provider: http_client_provider.clone(), - npm_global_cache_dir, + npm_cache_dir, cache_setting, text_only_progress_bar: progress_bar, maybe_node_modules_path: None, @@ -561,41 +713,56 @@ pub async fn run( // this is only used for installing packages, which isn't necessary with deno compile NpmInstallDepsProvider::empty(), ), - // Packages from different registries are already inlined in the ESZip, - // so no need to create actual `.npmrc` configuration. npmrc: create_default_npmrc(), lifecycle_scripts: Default::default(), }, )) .await?; - (fs, npm_resolver, None) + (in_npm_pkg_checker, npm_resolver) } }; let has_node_modules_dir = npm_resolver.root_node_modules_path().is_some(); let node_resolver = Arc::new(NodeResolver::new( deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()), - npm_resolver.clone().into_npm_resolver(), + in_npm_pkg_checker.clone(), + npm_resolver.clone().into_npm_pkg_folder_resolver(), + pkg_json_resolver.clone(), + )); + let cjs_tracker = Arc::new(CjsTracker::new( + in_npm_pkg_checker.clone(), + pkg_json_resolver.clone(), + if metadata.unstable_config.detect_cjs { + IsCjsResolutionMode::ImplicitTypeCommonJs + } else if metadata.workspace_resolver.package_jsons.is_empty() { + IsCjsResolutionMode::Disabled + } else { + IsCjsResolutionMode::ExplicitTypeCommonJs + }, )); - let cjs_resolutions = Arc::new(CjsResolutionStore::default()); let cache_db = Caches::new(deno_dir_provider.clone()); let node_analysis_cache = NodeAnalysisCache::new(cache_db.node_analysis_db()); - let cli_node_resolver = Arc::new(CliNodeResolver::new( - cjs_resolutions.clone(), - fs.clone(), - node_resolver.clone(), - npm_resolver.clone(), - )); + let npm_req_resolver = + Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions { + byonm_resolver: (npm_resolver.clone()).into_maybe_byonm(), + fs: CliDenoResolverFs(fs.clone()), + in_npm_pkg_checker: in_npm_pkg_checker.clone(), + node_resolver: node_resolver.clone(), + npm_req_resolver: npm_resolver.clone().into_npm_req_resolver(), + })); let cjs_esm_code_analyzer = CliCjsCodeAnalyzer::new( node_analysis_cache, + cjs_tracker.clone(), fs.clone(), - cli_node_resolver.clone(), + None, ); let node_code_translator = Arc::new(NodeCodeTranslator::new( cjs_esm_code_analyzer, deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()), + in_npm_pkg_checker, node_resolver.clone(), - npm_resolver.clone().into_npm_resolver(), + npm_resolver.clone().into_npm_pkg_folder_resolver(), + pkg_json_resolver.clone(), )); let workspace_resolver = { let import_map = match metadata.workspace_resolver.import_map { @@ -646,38 +813,53 @@ pub async fn run( metadata.workspace_resolver.pkg_json_resolution, ) }; + let code_cache = match metadata.code_cache_key { + Some(code_cache_key) => Some(Arc::new(DenoCompileCodeCache::new( + root_path.with_file_name(format!( + "{}.cache", + root_path.file_name().unwrap().to_string_lossy() + )), + code_cache_key, + )) as Arc), + None => { + log::debug!("Code cache disabled."); + None + } + }; let module_loader_factory = StandaloneModuleLoaderFactory { shared: Arc::new(SharedModuleLoaderState { - eszip: WorkspaceEszip { - eszip, - root_dir_url, - }, - workspace_resolver, - node_resolver: cli_node_resolver.clone(), + cjs_tracker: cjs_tracker.clone(), + code_cache: code_cache.clone(), + fs: fs.clone(), + modules, + node_code_translator: node_code_translator.clone(), + node_resolver: node_resolver.clone(), npm_module_loader: Arc::new(NpmModuleLoader::new( - cjs_resolutions, - node_code_translator, + cjs_tracker.clone(), fs.clone(), - cli_node_resolver, + node_code_translator, )), + npm_resolver: npm_resolver.clone(), + npm_req_resolver, + vfs, + workspace_resolver, }), }; let permissions = { - let mut permissions = metadata.permissions.to_options(); - // if running with an npm vfs, grant read access to it - if let Some(vfs_root) = maybe_vfs_root { - match &mut permissions.allow_read { - Some(vec) if vec.is_empty() => { - // do nothing, already granted - } - Some(vec) => { - vec.push(vfs_root.to_string_lossy().to_string()); - } - None => { - permissions.allow_read = - Some(vec![vfs_root.to_string_lossy().to_string()]); - } + let mut permissions = + metadata.permissions.to_options(/* cli_arg_urls */ &[]); + // grant read access to the vfs + match &mut permissions.allow_read { + Some(vec) if vec.is_empty() => { + // do nothing, already granted + } + Some(vec) => { + vec.push(root_path.to_string_lossy().to_string()); + } + None => { + permissions.allow_read = + Some(vec![root_path.to_string_lossy().to_string()]); } } @@ -697,12 +879,9 @@ pub async fn run( } checker }); - let permission_desc_parser = - Arc::new(RuntimePermissionDescriptorParser::new(fs.clone())); let worker_factory = CliMainWorkerFactory::new( Arc::new(BlobStore::default()), - // Code cache is not supported for standalone binary yet. - None, + code_cache, feature_checker, fs, None, @@ -711,8 +890,9 @@ pub async fn run( Box::new(module_loader_factory), node_resolver, npm_resolver, - permission_desc_parser, + pkg_json_resolver, root_cert_store_provider, + permissions, StorageKeyResolver::empty(), crate::args::DenoSubcommand::Run(Default::default()), CliMainWorkerOptions { @@ -726,7 +906,6 @@ pub async fn run( inspect_wait: false, strace_ops: None, is_inspecting: false, - is_npm_main: main_module.scheme() == "npm", skip_op_registration: true, location: metadata.location, argv0: NpmPackageReqReference::from_specifier(&main_module) @@ -744,6 +923,7 @@ pub async fn run( serve_port: None, serve_host: None, }, + metadata.otel_config, ); // Initialize v8 once from the main thread. @@ -752,7 +932,7 @@ pub async fn run( deno_core::JsRuntime::init_platform(None, true); let mut worker = worker_factory - .create_main_worker(WorkerExecutionMode::Run, main_module, permissions) + .create_main_worker(WorkerExecutionMode::Run, main_module) .await?; let exit_code = worker.run().await?; diff --git a/cli/standalone/serialization.rs b/cli/standalone/serialization.rs new file mode 100644 index 00000000000000..a5eb649bfdfec7 --- /dev/null +++ b/cli/standalone/serialization.rs @@ -0,0 +1,661 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::borrow::Cow; +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::io::Write; + +use deno_ast::MediaType; +use deno_core::anyhow::bail; +use deno_core::anyhow::Context; +use deno_core::error::AnyError; +use deno_core::serde_json; +use deno_core::url::Url; +use deno_core::FastString; +use deno_core::ModuleSourceCode; +use deno_core::ModuleType; +use deno_npm::resolution::SerializedNpmResolutionSnapshot; +use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; +use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; +use deno_npm::NpmPackageId; +use deno_semver::package::PackageReq; + +use crate::standalone::virtual_fs::VirtualDirectory; + +use super::binary::Metadata; +use super::virtual_fs::VfsBuilder; + +const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd"; + +/// Binary format: +/// * d3n0l4nd +/// * +/// * +/// * +/// * +/// * +/// * d3n0l4nd +pub fn serialize_binary_data_section( + metadata: &Metadata, + npm_snapshot: Option, + remote_modules: &RemoteModulesStoreBuilder, + vfs: VfsBuilder, +) -> Result, AnyError> { + fn write_bytes_with_len(bytes: &mut Vec, data: &[u8]) { + bytes.extend_from_slice(&(data.len() as u64).to_le_bytes()); + bytes.extend_from_slice(data); + } + + let mut bytes = Vec::new(); + bytes.extend_from_slice(MAGIC_BYTES); + + // 1. Metadata + { + let metadata = serde_json::to_string(metadata)?; + write_bytes_with_len(&mut bytes, metadata.as_bytes()); + } + // 2. Npm snapshot + { + let npm_snapshot = + npm_snapshot.map(serialize_npm_snapshot).unwrap_or_default(); + write_bytes_with_len(&mut bytes, &npm_snapshot); + } + // 3. Remote modules + { + let update_index = bytes.len(); + bytes.extend_from_slice(&(0_u64).to_le_bytes()); + let start_index = bytes.len(); + remote_modules.write(&mut bytes)?; + let length = bytes.len() - start_index; + let length_bytes = (length as u64).to_le_bytes(); + bytes[update_index..update_index + length_bytes.len()] + .copy_from_slice(&length_bytes); + } + // 4. VFS + { + let (vfs, vfs_files) = vfs.into_dir_and_files(); + let vfs = serde_json::to_string(&vfs)?; + write_bytes_with_len(&mut bytes, vfs.as_bytes()); + let vfs_bytes_len = vfs_files.iter().map(|f| f.len() as u64).sum::(); + bytes.extend_from_slice(&vfs_bytes_len.to_le_bytes()); + for file in &vfs_files { + bytes.extend_from_slice(file); + } + } + + // write the magic bytes at the end so we can use it + // to make sure we've deserialized correctly + bytes.extend_from_slice(MAGIC_BYTES); + + Ok(bytes) +} + +pub struct DeserializedDataSection { + pub metadata: Metadata, + pub npm_snapshot: Option, + pub remote_modules: RemoteModulesStore, + pub vfs_dir: VirtualDirectory, + pub vfs_files_data: &'static [u8], +} + +pub fn deserialize_binary_data_section( + data: &'static [u8], +) -> Result, AnyError> { + fn read_bytes_with_len(input: &[u8]) -> Result<(&[u8], &[u8]), AnyError> { + let (input, len) = read_u64(input)?; + let (input, data) = read_bytes(input, len as usize)?; + Ok((input, data)) + } + + fn read_magic_bytes(input: &[u8]) -> Result<(&[u8], bool), AnyError> { + if input.len() < MAGIC_BYTES.len() { + bail!("Unexpected end of data. Could not find magic bytes."); + } + let (magic_bytes, input) = input.split_at(MAGIC_BYTES.len()); + if magic_bytes != MAGIC_BYTES { + return Ok((input, false)); + } + Ok((input, true)) + } + + let (input, found) = read_magic_bytes(data)?; + if !found { + return Ok(None); + } + + // 1. Metadata + let (input, data) = read_bytes_with_len(input).context("reading metadata")?; + let metadata: Metadata = + serde_json::from_slice(data).context("deserializing metadata")?; + // 2. Npm snapshot + let (input, data) = + read_bytes_with_len(input).context("reading npm snapshot")?; + let npm_snapshot = if data.is_empty() { + None + } else { + Some(deserialize_npm_snapshot(data).context("deserializing npm snapshot")?) + }; + // 3. Remote modules + let (input, data) = + read_bytes_with_len(input).context("reading remote modules data")?; + let remote_modules = + RemoteModulesStore::build(data).context("deserializing remote modules")?; + // 4. VFS + let (input, data) = read_bytes_with_len(input).context("vfs")?; + let vfs_dir: VirtualDirectory = + serde_json::from_slice(data).context("deserializing vfs data")?; + let (input, vfs_files_data) = + read_bytes_with_len(input).context("reading vfs files data")?; + + // finally ensure we read the magic bytes at the end + let (_input, found) = read_magic_bytes(input)?; + if !found { + bail!("Could not find magic bytes at the end of the data."); + } + + Ok(Some(DeserializedDataSection { + metadata, + npm_snapshot, + remote_modules, + vfs_dir, + vfs_files_data, + })) +} + +#[derive(Default)] +pub struct RemoteModulesStoreBuilder { + specifiers: Vec<(String, u64)>, + data: Vec<(MediaType, Vec)>, + data_byte_len: u64, + redirects: Vec<(String, String)>, + redirects_len: u64, +} + +impl RemoteModulesStoreBuilder { + pub fn add(&mut self, specifier: &Url, media_type: MediaType, data: Vec) { + log::debug!("Adding '{}' ({})", specifier, media_type); + let specifier = specifier.to_string(); + self.specifiers.push((specifier, self.data_byte_len)); + self.data_byte_len += 1 + 8 + data.len() as u64; // media type (1 byte), data length (8 bytes), data + self.data.push((media_type, data)); + } + + pub fn add_redirects(&mut self, redirects: &BTreeMap) { + self.redirects.reserve(redirects.len()); + for (from, to) in redirects { + log::debug!("Adding redirect '{}' -> '{}'", from, to); + let from = from.to_string(); + let to = to.to_string(); + self.redirects_len += (4 + from.len() + 4 + to.len()) as u64; + self.redirects.push((from, to)); + } + } + + fn write(&self, writer: &mut dyn Write) -> Result<(), AnyError> { + writer.write_all(&(self.specifiers.len() as u32).to_le_bytes())?; + writer.write_all(&(self.redirects.len() as u32).to_le_bytes())?; + for (specifier, offset) in &self.specifiers { + writer.write_all(&(specifier.len() as u32).to_le_bytes())?; + writer.write_all(specifier.as_bytes())?; + writer.write_all(&offset.to_le_bytes())?; + } + for (from, to) in &self.redirects { + writer.write_all(&(from.len() as u32).to_le_bytes())?; + writer.write_all(from.as_bytes())?; + writer.write_all(&(to.len() as u32).to_le_bytes())?; + writer.write_all(to.as_bytes())?; + } + for (media_type, data) in &self.data { + writer.write_all(&[serialize_media_type(*media_type)])?; + writer.write_all(&(data.len() as u64).to_le_bytes())?; + writer.write_all(data)?; + } + Ok(()) + } +} + +pub enum DenoCompileModuleSource { + String(&'static str), + Bytes(Cow<'static, [u8]>), +} + +impl DenoCompileModuleSource { + pub fn into_for_v8(self) -> ModuleSourceCode { + fn into_bytes(data: Cow<'static, [u8]>) -> ModuleSourceCode { + ModuleSourceCode::Bytes(match data { + Cow::Borrowed(d) => d.into(), + Cow::Owned(d) => d.into_boxed_slice().into(), + }) + } + + match self { + // todo(https://github.com/denoland/deno_core/pull/943): store whether + // the string is ascii or not ahead of time so we can avoid the is_ascii() + // check in FastString::from_static + Self::String(s) => ModuleSourceCode::String(FastString::from_static(s)), + Self::Bytes(b) => into_bytes(b), + } + } +} + +pub struct DenoCompileModuleData<'a> { + pub specifier: &'a Url, + pub media_type: MediaType, + pub data: Cow<'static, [u8]>, +} + +impl<'a> DenoCompileModuleData<'a> { + pub fn into_parts(self) -> (&'a Url, ModuleType, DenoCompileModuleSource) { + fn into_string_unsafe(data: Cow<'static, [u8]>) -> DenoCompileModuleSource { + match data { + Cow::Borrowed(d) => DenoCompileModuleSource::String( + // SAFETY: we know this is a valid utf8 string + unsafe { std::str::from_utf8_unchecked(d) }, + ), + Cow::Owned(d) => DenoCompileModuleSource::Bytes(Cow::Owned(d)), + } + } + + let (media_type, source) = match self.media_type { + MediaType::JavaScript + | MediaType::Jsx + | MediaType::Mjs + | MediaType::Cjs + | MediaType::TypeScript + | MediaType::Mts + | MediaType::Cts + | MediaType::Dts + | MediaType::Dmts + | MediaType::Dcts + | MediaType::Tsx => { + (ModuleType::JavaScript, into_string_unsafe(self.data)) + } + MediaType::Json => (ModuleType::Json, into_string_unsafe(self.data)), + MediaType::Wasm => { + (ModuleType::Wasm, DenoCompileModuleSource::Bytes(self.data)) + } + // just assume javascript if we made it here + MediaType::Css | MediaType::SourceMap | MediaType::Unknown => ( + ModuleType::JavaScript, + DenoCompileModuleSource::Bytes(self.data), + ), + }; + (self.specifier, media_type, source) + } +} + +enum RemoteModulesStoreSpecifierValue { + Data(usize), + Redirect(Url), +} + +pub struct RemoteModulesStore { + specifiers: HashMap, + files_data: &'static [u8], +} + +impl RemoteModulesStore { + fn build(data: &'static [u8]) -> Result { + fn read_specifier(input: &[u8]) -> Result<(&[u8], (Url, u64)), AnyError> { + let (input, specifier) = read_string_lossy(input)?; + let specifier = Url::parse(&specifier)?; + let (input, offset) = read_u64(input)?; + Ok((input, (specifier, offset))) + } + + fn read_redirect(input: &[u8]) -> Result<(&[u8], (Url, Url)), AnyError> { + let (input, from) = read_string_lossy(input)?; + let from = Url::parse(&from)?; + let (input, to) = read_string_lossy(input)?; + let to = Url::parse(&to)?; + Ok((input, (from, to))) + } + + fn read_headers( + input: &[u8], + ) -> Result<(&[u8], HashMap), AnyError> + { + let (input, specifiers_len) = read_u32_as_usize(input)?; + let (mut input, redirects_len) = read_u32_as_usize(input)?; + let mut specifiers = + HashMap::with_capacity(specifiers_len + redirects_len); + for _ in 0..specifiers_len { + let (current_input, (specifier, offset)) = + read_specifier(input).context("reading specifier")?; + input = current_input; + specifiers.insert( + specifier, + RemoteModulesStoreSpecifierValue::Data(offset as usize), + ); + } + + for _ in 0..redirects_len { + let (current_input, (from, to)) = read_redirect(input)?; + input = current_input; + specifiers.insert(from, RemoteModulesStoreSpecifierValue::Redirect(to)); + } + + Ok((input, specifiers)) + } + + let (files_data, specifiers) = read_headers(data)?; + + Ok(Self { + specifiers, + files_data, + }) + } + + pub fn resolve_specifier<'a>( + &'a self, + specifier: &'a Url, + ) -> Result, AnyError> { + let mut count = 0; + let mut current = specifier; + loop { + if count > 10 { + bail!("Too many redirects resolving '{}'", specifier); + } + match self.specifiers.get(current) { + Some(RemoteModulesStoreSpecifierValue::Redirect(to)) => { + current = to; + count += 1; + } + Some(RemoteModulesStoreSpecifierValue::Data(_)) => { + return Ok(Some(current)); + } + None => { + return Ok(None); + } + } + } + } + + pub fn read<'a>( + &'a self, + original_specifier: &'a Url, + ) -> Result>, AnyError> { + let mut count = 0; + let mut specifier = original_specifier; + loop { + if count > 10 { + bail!("Too many redirects resolving '{}'", original_specifier); + } + match self.specifiers.get(specifier) { + Some(RemoteModulesStoreSpecifierValue::Redirect(to)) => { + specifier = to; + count += 1; + } + Some(RemoteModulesStoreSpecifierValue::Data(offset)) => { + let input = &self.files_data[*offset..]; + let (input, media_type_byte) = read_bytes(input, 1)?; + let media_type = deserialize_media_type(media_type_byte[0])?; + let (input, len) = read_u64(input)?; + let (_input, data) = read_bytes(input, len as usize)?; + return Ok(Some(DenoCompileModuleData { + specifier, + media_type, + data: Cow::Borrowed(data), + })); + } + None => { + return Ok(None); + } + } + } + } +} + +fn serialize_npm_snapshot( + mut snapshot: SerializedNpmResolutionSnapshot, +) -> Vec { + fn append_string(bytes: &mut Vec, string: &str) { + let len = string.len() as u32; + bytes.extend_from_slice(&len.to_le_bytes()); + bytes.extend_from_slice(string.as_bytes()); + } + + snapshot.packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism + let ids_to_stored_ids = snapshot + .packages + .iter() + .enumerate() + .map(|(i, pkg)| (&pkg.id, i as u32)) + .collect::>(); + + let mut root_packages: Vec<_> = snapshot.root_packages.iter().collect(); + root_packages.sort(); + let mut bytes = Vec::new(); + + bytes.extend_from_slice(&(snapshot.packages.len() as u32).to_le_bytes()); + for pkg in &snapshot.packages { + append_string(&mut bytes, &pkg.id.as_serialized()); + } + + bytes.extend_from_slice(&(root_packages.len() as u32).to_le_bytes()); + for (req, id) in root_packages { + append_string(&mut bytes, &req.to_string()); + let id = ids_to_stored_ids.get(&id).unwrap(); + bytes.extend_from_slice(&id.to_le_bytes()); + } + + for pkg in &snapshot.packages { + let deps_len = pkg.dependencies.len() as u32; + bytes.extend_from_slice(&deps_len.to_le_bytes()); + let mut deps: Vec<_> = pkg.dependencies.iter().collect(); + deps.sort(); + for (req, id) in deps { + append_string(&mut bytes, req); + let id = ids_to_stored_ids.get(&id).unwrap(); + bytes.extend_from_slice(&id.to_le_bytes()); + } + } + + bytes +} + +fn deserialize_npm_snapshot( + input: &[u8], +) -> Result { + fn parse_id(input: &[u8]) -> Result<(&[u8], NpmPackageId), AnyError> { + let (input, id) = read_string_lossy(input)?; + let id = NpmPackageId::from_serialized(&id)?; + Ok((input, id)) + } + + #[allow(clippy::needless_lifetimes)] // clippy bug + fn parse_root_package<'a>( + id_to_npm_id: &'a impl Fn(usize) -> Result, + ) -> impl Fn(&[u8]) -> Result<(&[u8], (PackageReq, NpmPackageId)), AnyError> + 'a + { + |input| { + let (input, req) = read_string_lossy(input)?; + let req = PackageReq::from_str(&req)?; + let (input, id) = read_u32_as_usize(input)?; + Ok((input, (req, id_to_npm_id(id)?))) + } + } + + #[allow(clippy::needless_lifetimes)] // clippy bug + fn parse_package_dep<'a>( + id_to_npm_id: &'a impl Fn(usize) -> Result, + ) -> impl Fn(&[u8]) -> Result<(&[u8], (String, NpmPackageId)), AnyError> + 'a + { + |input| { + let (input, req) = read_string_lossy(input)?; + let (input, id) = read_u32_as_usize(input)?; + Ok((input, (req.into_owned(), id_to_npm_id(id)?))) + } + } + + fn parse_package<'a>( + input: &'a [u8], + id: NpmPackageId, + id_to_npm_id: &impl Fn(usize) -> Result, + ) -> Result<(&'a [u8], SerializedNpmResolutionSnapshotPackage), AnyError> { + let (input, deps_len) = read_u32_as_usize(input)?; + let (input, dependencies) = + parse_hashmap_n_times(input, deps_len, parse_package_dep(id_to_npm_id))?; + Ok(( + input, + SerializedNpmResolutionSnapshotPackage { + id, + system: Default::default(), + dist: Default::default(), + dependencies, + optional_dependencies: Default::default(), + bin: None, + scripts: Default::default(), + deprecated: Default::default(), + }, + )) + } + + let (input, packages_len) = read_u32_as_usize(input)?; + + // get a hashmap of all the npm package ids to their serialized ids + let (input, data_ids_to_npm_ids) = + parse_vec_n_times(input, packages_len, parse_id) + .context("deserializing id")?; + let data_id_to_npm_id = |id: usize| { + data_ids_to_npm_ids + .get(id) + .cloned() + .ok_or_else(|| deno_core::anyhow::anyhow!("Invalid npm package id")) + }; + + let (input, root_packages_len) = read_u32_as_usize(input)?; + let (input, root_packages) = parse_hashmap_n_times( + input, + root_packages_len, + parse_root_package(&data_id_to_npm_id), + ) + .context("deserializing root package")?; + let (input, packages) = + parse_vec_n_times_with_index(input, packages_len, |input, index| { + parse_package(input, data_id_to_npm_id(index)?, &data_id_to_npm_id) + }) + .context("deserializing package")?; + + if !input.is_empty() { + bail!("Unexpected data left over"); + } + + Ok( + SerializedNpmResolutionSnapshot { + packages, + root_packages, + } + // this is ok because we have already verified that all the + // identifiers found in the snapshot are valid via the + // npm package id -> npm package id mapping + .into_valid_unsafe(), + ) +} + +fn serialize_media_type(media_type: MediaType) -> u8 { + match media_type { + MediaType::JavaScript => 0, + MediaType::Jsx => 1, + MediaType::Mjs => 2, + MediaType::Cjs => 3, + MediaType::TypeScript => 4, + MediaType::Mts => 5, + MediaType::Cts => 6, + MediaType::Dts => 7, + MediaType::Dmts => 8, + MediaType::Dcts => 9, + MediaType::Tsx => 10, + MediaType::Json => 11, + MediaType::Wasm => 12, + MediaType::Css => 13, + MediaType::SourceMap => 14, + MediaType::Unknown => 15, + } +} + +fn deserialize_media_type(value: u8) -> Result { + match value { + 0 => Ok(MediaType::JavaScript), + 1 => Ok(MediaType::Jsx), + 2 => Ok(MediaType::Mjs), + 3 => Ok(MediaType::Cjs), + 4 => Ok(MediaType::TypeScript), + 5 => Ok(MediaType::Mts), + 6 => Ok(MediaType::Cts), + 7 => Ok(MediaType::Dts), + 8 => Ok(MediaType::Dmts), + 9 => Ok(MediaType::Dcts), + 10 => Ok(MediaType::Tsx), + 11 => Ok(MediaType::Json), + 12 => Ok(MediaType::Wasm), + 13 => Ok(MediaType::Css), + 14 => Ok(MediaType::SourceMap), + 15 => Ok(MediaType::Unknown), + _ => bail!("Unknown media type value: {}", value), + } +} + +fn parse_hashmap_n_times( + mut input: &[u8], + times: usize, + parse: impl Fn(&[u8]) -> Result<(&[u8], (TKey, TValue)), AnyError>, +) -> Result<(&[u8], HashMap), AnyError> { + let mut results = HashMap::with_capacity(times); + for _ in 0..times { + let result = parse(input); + let (new_input, (key, value)) = result?; + results.insert(key, value); + input = new_input; + } + Ok((input, results)) +} + +fn parse_vec_n_times( + input: &[u8], + times: usize, + parse: impl Fn(&[u8]) -> Result<(&[u8], TResult), AnyError>, +) -> Result<(&[u8], Vec), AnyError> { + parse_vec_n_times_with_index(input, times, |input, _index| parse(input)) +} + +fn parse_vec_n_times_with_index( + mut input: &[u8], + times: usize, + parse: impl Fn(&[u8], usize) -> Result<(&[u8], TResult), AnyError>, +) -> Result<(&[u8], Vec), AnyError> { + let mut results = Vec::with_capacity(times); + for i in 0..times { + let result = parse(input, i); + let (new_input, result) = result?; + results.push(result); + input = new_input; + } + Ok((input, results)) +} + +fn read_bytes(input: &[u8], len: usize) -> Result<(&[u8], &[u8]), AnyError> { + if input.len() < len { + bail!("Unexpected end of data.",); + } + let (len_bytes, input) = input.split_at(len); + Ok((input, len_bytes)) +} + +fn read_string_lossy(input: &[u8]) -> Result<(&[u8], Cow), AnyError> { + let (input, str_len) = read_u32_as_usize(input)?; + let (input, data_bytes) = read_bytes(input, str_len)?; + Ok((input, String::from_utf8_lossy(data_bytes))) +} + +fn read_u32_as_usize(input: &[u8]) -> Result<(&[u8], usize), AnyError> { + let (input, len_bytes) = read_bytes(input, 4)?; + let len = u32::from_le_bytes(len_bytes.try_into()?); + Ok((input, len as usize)) +} + +fn read_u64(input: &[u8]) -> Result<(&[u8], u64), AnyError> { + let (input, len_bytes) = read_bytes(input, 8)?; + let len = u64::from_le_bytes(len_bytes.try_into()?); + Ok((input, len)) +} diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs index 53d045b6257614..66fc835534436d 100644 --- a/cli/standalone/virtual_fs.rs +++ b/cli/standalone/virtual_fs.rs @@ -7,6 +7,7 @@ use std::fs::File; use std::io::Read; use std::io::Seek; use std::io::SeekFrom; +use std::ops::Range; use std::path::Path; use std::path::PathBuf; use std::rc::Rc; @@ -31,6 +32,15 @@ use thiserror::Error; use crate::util; use crate::util::fs::canonicalize_path; +#[derive(Debug, Copy, Clone)] +pub enum VfsFileSubDataKind { + /// Raw bytes of the file. + Raw, + /// Bytes to use for module loading. For example, for TypeScript + /// files this will be the transpiled JavaScript source. + ModuleGraph, +} + #[derive(Error, Debug)] #[error( "Failed to strip prefix '{}' from '{}'", root_path.display(), target.display() @@ -50,7 +60,8 @@ pub struct VfsBuilder { impl VfsBuilder { pub fn new(root_path: PathBuf) -> Result { - let root_path = canonicalize_path(&root_path)?; + let root_path = canonicalize_path(&root_path) + .with_context(|| format!("Canonicalizing {}", root_path.display()))?; log::debug!("Building vfs with root '{}'", root_path.display()); Ok(Self { root_dir: VirtualDirectory { @@ -67,6 +78,26 @@ impl VfsBuilder { }) } + pub fn set_new_root_path( + &mut self, + root_path: PathBuf, + ) -> Result<(), AnyError> { + let root_path = canonicalize_path(&root_path)?; + self.root_path = root_path; + self.root_dir = VirtualDirectory { + name: self + .root_path + .file_stem() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or("root".to_string()), + entries: vec![VfsEntry::Dir(VirtualDirectory { + name: std::mem::take(&mut self.root_dir.name), + entries: std::mem::take(&mut self.root_dir.entries), + })], + }; + Ok(()) + } + pub fn with_root_dir( &mut self, with_root: impl FnOnce(&mut VirtualDirectory) -> R, @@ -119,7 +150,11 @@ impl VfsBuilder { // inline the symlink and make the target file let file_bytes = std::fs::read(&target) .with_context(|| format!("Reading {}", path.display()))?; - self.add_file(&path, file_bytes)?; + self.add_file_with_data_inner( + &path, + file_bytes, + VfsFileSubDataKind::Raw, + )?; } else { log::warn!( "{} Symlink target is outside '{}'. Excluding symlink at '{}' with target '{}'.", @@ -191,16 +226,34 @@ impl VfsBuilder { self.add_file_at_path_not_symlink(&target_path) } - pub fn add_file_at_path_not_symlink( + fn add_file_at_path_not_symlink( &mut self, path: &Path, ) -> Result<(), AnyError> { let file_bytes = std::fs::read(path) .with_context(|| format!("Reading {}", path.display()))?; - self.add_file(path, file_bytes) + self.add_file_with_data_inner(path, file_bytes, VfsFileSubDataKind::Raw) + } + + pub fn add_file_with_data( + &mut self, + path: &Path, + data: Vec, + sub_data_kind: VfsFileSubDataKind, + ) -> Result<(), AnyError> { + let target_path = canonicalize_path(path)?; + if target_path != path { + self.add_symlink(path, &target_path)?; + } + self.add_file_with_data_inner(&target_path, data, sub_data_kind) } - fn add_file(&mut self, path: &Path, data: Vec) -> Result<(), AnyError> { + fn add_file_with_data_inner( + &mut self, + path: &Path, + data: Vec, + sub_data_kind: VfsFileSubDataKind, + ) -> Result<(), AnyError> { log::debug!("Adding file '{}'", path.display()); let checksum = util::checksum::gen(&[&data]); let offset = if let Some(offset) = self.file_offsets.get(&checksum) { @@ -215,8 +268,19 @@ impl VfsBuilder { let name = path.file_name().unwrap().to_string_lossy(); let data_len = data.len(); match dir.entries.binary_search_by(|e| e.name().cmp(&name)) { - Ok(_) => { - // already added, just ignore + Ok(index) => { + let entry = &mut dir.entries[index]; + match entry { + VfsEntry::File(virtual_file) => match sub_data_kind { + VfsFileSubDataKind::Raw => { + virtual_file.offset = offset; + } + VfsFileSubDataKind::ModuleGraph => { + virtual_file.module_graph_offset = offset; + } + }, + VfsEntry::Dir(_) | VfsEntry::Symlink(_) => unreachable!(), + } } Err(insert_index) => { dir.entries.insert( @@ -224,6 +288,7 @@ impl VfsBuilder { VfsEntry::File(VirtualFile { name: name.to_string(), offset, + module_graph_offset: offset, len: data.len() as u64, }), ); @@ -249,29 +314,36 @@ impl VfsBuilder { path.display(), target.display() ); - let dest = self.path_relative_root(target)?; - if dest == self.path_relative_root(path)? { + let relative_target = self.path_relative_root(target)?; + let relative_path = match self.path_relative_root(path) { + Ok(path) => path, + Err(StripRootError { .. }) => { + // ignore if the original path is outside the root directory + return Ok(()); + } + }; + if relative_target == relative_path { // it's the same, ignore return Ok(()); } let dir = self.add_dir(path.parent().unwrap())?; let name = path.file_name().unwrap().to_string_lossy(); match dir.entries.binary_search_by(|e| e.name().cmp(&name)) { - Ok(_) => unreachable!(), + Ok(_) => Ok(()), // previously inserted Err(insert_index) => { dir.entries.insert( insert_index, VfsEntry::Symlink(VirtualSymlink { name: name.to_string(), - dest_parts: dest + dest_parts: relative_target .components() .map(|c| c.as_os_str().to_string_lossy().to_string()) .collect::>(), }), ); + Ok(()) } } - Ok(()) } pub fn into_dir_and_files(self) -> (VirtualDirectory, Vec>) { @@ -306,6 +378,7 @@ impl<'a> VfsEntryRef<'a> { atime: None, birthtime: None, mtime: None, + ctime: None, blksize: 0, size: 0, dev: 0, @@ -328,6 +401,7 @@ impl<'a> VfsEntryRef<'a> { atime: None, birthtime: None, mtime: None, + ctime: None, blksize: 0, size: file.len, dev: 0, @@ -350,6 +424,7 @@ impl<'a> VfsEntryRef<'a> { atime: None, birthtime: None, mtime: None, + ctime: None, blksize: 0, size: 0, dev: 0, @@ -406,6 +481,12 @@ pub struct VirtualDirectory { pub struct VirtualFile { pub name: String, pub offset: u64, + /// Offset file to use for module loading when it differs from the + /// raw file. Often this will be the same offset as above for data + /// such as JavaScript files, but for TypeScript files the `offset` + /// will be the original raw bytes when included as an asset and this + /// offset will be to the transpiled JavaScript source. + pub module_graph_offset: u64, pub len: u64, } @@ -586,7 +667,7 @@ impl FileBackedVfsFile { } fn read_to_buf(&self, buf: &mut [u8]) -> FsResult { - let pos = { + let read_pos = { let mut pos = self.pos.lock(); let read_pos = *pos; // advance the position due to the read @@ -595,12 +676,12 @@ impl FileBackedVfsFile { }; self .vfs - .read_file(&self.file, pos, buf) + .read_file(&self.file, read_pos, buf) .map_err(|err| err.into()) } - fn read_to_end(&self) -> FsResult> { - let pos = { + fn read_to_end(&self) -> FsResult> { + let read_pos = { let mut pos = self.pos.lock(); let read_pos = *pos; // todo(dsherret): should this always set it to the end of the file? @@ -610,13 +691,21 @@ impl FileBackedVfsFile { } read_pos }; - if pos > self.file.len { - return Ok(Vec::new()); + if read_pos > self.file.len { + return Ok(Cow::Borrowed(&[])); + } + if read_pos == 0 { + Ok( + self + .vfs + .read_file_all(&self.file, VfsFileSubDataKind::Raw)?, + ) + } else { + let size = (self.file.len - read_pos) as usize; + let mut buf = vec![0; size]; + self.vfs.read_file(&self.file, read_pos, &mut buf)?; + Ok(Cow::Owned(buf)) } - let size = (self.file.len - pos) as usize; - let mut buf = vec![0; size]; - self.vfs.read_file(&self.file, pos, &mut buf)?; - Ok(buf) } } @@ -654,10 +743,10 @@ impl deno_io::fs::File for FileBackedVfsFile { Err(FsError::NotSupported) } - fn read_all_sync(self: Rc) -> FsResult> { + fn read_all_sync(self: Rc) -> FsResult> { self.read_to_end() } - async fn read_all_async(self: Rc) -> FsResult> { + async fn read_all_async(self: Rc) -> FsResult> { let inner = (*self).clone(); tokio::task::spawn_blocking(move || inner.read_to_end()).await? } @@ -751,14 +840,14 @@ impl deno_io::fs::File for FileBackedVfsFile { #[derive(Debug)] pub struct FileBackedVfs { - file: Mutex>, + vfs_data: Cow<'static, [u8]>, fs_root: VfsRoot, } impl FileBackedVfs { - pub fn new(file: Vec, fs_root: VfsRoot) -> Self { + pub fn new(data: Cow<'static, [u8]>, fs_root: VfsRoot) -> Self { Self { - file: Mutex::new(file), + vfs_data: data, fs_root, } } @@ -827,10 +916,16 @@ impl FileBackedVfs { Ok(path) } - pub fn read_file_all(&self, file: &VirtualFile) -> std::io::Result> { - let mut buf = vec![0; file.len as usize]; - self.read_file(file, 0, &mut buf)?; - Ok(buf) + pub fn read_file_all( + &self, + file: &VirtualFile, + sub_data_kind: VfsFileSubDataKind, + ) -> std::io::Result> { + let read_range = self.get_read_range(file, sub_data_kind, 0, file.len)?; + match &self.vfs_data { + Cow::Borrowed(data) => Ok(Cow::Borrowed(&data[read_range])), + Cow::Owned(data) => Ok(Cow::Owned(data[read_range].to_vec())), + } } pub fn read_file( @@ -839,18 +934,38 @@ impl FileBackedVfs { pos: u64, buf: &mut [u8], ) -> std::io::Result { - let data = self.file.lock(); - let start = self.fs_root.start_file_offset + file.offset + pos; - let end = start + buf.len() as u64; - if end > data.len() as u64 { + let read_range = self.get_read_range( + file, + VfsFileSubDataKind::Raw, + pos, + buf.len() as u64, + )?; + let read_len = read_range.len(); + buf[..read_len].copy_from_slice(&self.vfs_data[read_range]); + Ok(read_len) + } + + fn get_read_range( + &self, + file: &VirtualFile, + sub_data_kind: VfsFileSubDataKind, + pos: u64, + len: u64, + ) -> std::io::Result> { + if pos > file.len { return Err(std::io::Error::new( std::io::ErrorKind::UnexpectedEof, "unexpected EOF", )); } - - buf.copy_from_slice(&data[start as usize..end as usize]); - Ok(buf.len()) + let offset = match sub_data_kind { + VfsFileSubDataKind::Raw => file.offset, + VfsFileSubDataKind::ModuleGraph => file.module_graph_offset, + }; + let file_offset = self.fs_root.start_file_offset + offset; + let start = file_offset + pos; + let end = file_offset + std::cmp::min(pos + len, file.len); + Ok(start as usize..end as usize) } pub fn dir_entry(&self, path: &Path) -> std::io::Result<&VirtualDirectory> { @@ -888,7 +1003,13 @@ mod test { #[track_caller] fn read_file(vfs: &FileBackedVfs, path: &Path) -> String { let file = vfs.file_entry(path).unwrap(); - String::from_utf8(vfs.read_file_all(file).unwrap()).unwrap() + String::from_utf8( + vfs + .read_file_all(file, VfsFileSubDataKind::Raw) + .unwrap() + .into_owned(), + ) + .unwrap() } #[test] @@ -901,20 +1022,40 @@ mod test { let src_path = src_path.to_path_buf(); let mut builder = VfsBuilder::new(src_path.clone()).unwrap(); builder - .add_file(&src_path.join("a.txt"), "data".into()) + .add_file_with_data_inner( + &src_path.join("a.txt"), + "data".into(), + VfsFileSubDataKind::Raw, + ) .unwrap(); builder - .add_file(&src_path.join("b.txt"), "data".into()) + .add_file_with_data_inner( + &src_path.join("b.txt"), + "data".into(), + VfsFileSubDataKind::Raw, + ) .unwrap(); assert_eq!(builder.files.len(), 1); // because duplicate data builder - .add_file(&src_path.join("c.txt"), "c".into()) + .add_file_with_data_inner( + &src_path.join("c.txt"), + "c".into(), + VfsFileSubDataKind::Raw, + ) .unwrap(); builder - .add_file(&src_path.join("sub_dir").join("d.txt"), "d".into()) + .add_file_with_data_inner( + &src_path.join("sub_dir").join("d.txt"), + "d".into(), + VfsFileSubDataKind::Raw, + ) .unwrap(); builder - .add_file(&src_path.join("e.txt"), "e".into()) + .add_file_with_data_inner( + &src_path.join("e.txt"), + "e".into(), + VfsFileSubDataKind::Raw, + ) .unwrap(); builder .add_symlink( @@ -1031,7 +1172,7 @@ mod test { ( dest_path.to_path_buf(), FileBackedVfs::new( - data, + Cow::Owned(data), VfsRoot { dir: root_dir, root_path: dest_path.to_path_buf(), @@ -1082,9 +1223,10 @@ mod test { let temp_path = temp_dir.path().canonicalize(); let mut builder = VfsBuilder::new(temp_path.to_path_buf()).unwrap(); builder - .add_file( + .add_file_with_data_inner( temp_path.join("a.txt").as_path(), "0123456789".to_string().into_bytes(), + VfsFileSubDataKind::Raw, ) .unwrap(); let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir); diff --git a/cli/task_runner.rs b/cli/task_runner.rs index ab7163bc93b720..ec043f280e73cd 100644 --- a/cli/task_runner.rs +++ b/cli/task_runner.rs @@ -16,8 +16,11 @@ use deno_task_shell::ExecutableCommand; use deno_task_shell::ExecuteResult; use deno_task_shell::ShellCommand; use deno_task_shell::ShellCommandContext; +use deno_task_shell::ShellPipeReader; +use deno_task_shell::ShellPipeWriter; use lazy_regex::Lazy; use regex::Regex; +use tokio::task::JoinHandle; use tokio::task::LocalSet; use crate::npm::CliNpmResolver; @@ -36,6 +39,35 @@ pub fn get_script_with_args(script: &str, argv: &[String]) -> String { script.trim().to_owned() } +pub struct TaskStdio(Option, ShellPipeWriter); + +impl TaskStdio { + pub fn stdout() -> Self { + Self(None, ShellPipeWriter::stdout()) + } + pub fn stderr() -> Self { + Self(None, ShellPipeWriter::stderr()) + } + pub fn piped() -> Self { + let (r, w) = deno_task_shell::pipe(); + Self(Some(r), w) + } +} + +pub struct TaskIo { + pub stdout: TaskStdio, + pub stderr: TaskStdio, +} + +impl Default for TaskIo { + fn default() -> Self { + Self { + stderr: TaskStdio::stderr(), + stdout: TaskStdio::stdout(), + } + } +} + pub struct RunTaskOptions<'a> { pub task_name: &'a str, pub script: &'a str, @@ -45,24 +77,69 @@ pub struct RunTaskOptions<'a> { pub argv: &'a [String], pub custom_commands: HashMap>, pub root_node_modules_dir: Option<&'a Path>, + pub stdio: Option, } pub type TaskCustomCommands = HashMap>; -pub async fn run_task(opts: RunTaskOptions<'_>) -> Result { +pub struct TaskResult { + pub exit_code: i32, + pub stdout: Option>, + pub stderr: Option>, +} + +pub async fn run_task( + opts: RunTaskOptions<'_>, +) -> Result { let script = get_script_with_args(opts.script, opts.argv); let seq_list = deno_task_shell::parser::parse(&script) .with_context(|| format!("Error parsing script '{}'.", opts.task_name))?; let env_vars = prepare_env_vars(opts.env_vars, opts.init_cwd, opts.root_node_modules_dir); + let state = + deno_task_shell::ShellState::new(env_vars, opts.cwd, opts.custom_commands); + let stdio = opts.stdio.unwrap_or_default(); + let ( + TaskStdio(stdout_read, stdout_write), + TaskStdio(stderr_read, stderr_write), + ) = (stdio.stdout, stdio.stderr); + + fn read(reader: ShellPipeReader) -> JoinHandle, AnyError>> { + tokio::task::spawn_blocking(move || { + let mut buf = Vec::new(); + reader.pipe_to(&mut buf)?; + Ok(buf) + }) + } + + let stdout = stdout_read.map(read); + let stderr = stderr_read.map(read); + let local = LocalSet::new(); - let future = deno_task_shell::execute( - seq_list, - env_vars, - opts.cwd, - opts.custom_commands, - ); - Ok(local.run_until(future).await) + let future = async move { + let exit_code = deno_task_shell::execute_with_pipes( + seq_list, + state, + ShellPipeReader::stdin(), + stdout_write, + stderr_write, + ) + .await; + Ok::<_, AnyError>(TaskResult { + exit_code, + stdout: if let Some(stdout) = stdout { + Some(stdout.await??) + } else { + None + }, + stderr: if let Some(stderr) = stderr { + Some(stderr.await??) + } else { + None + }, + }) + }; + local.run_until(future).await } fn prepare_env_vars( @@ -78,6 +155,12 @@ fn prepare_env_vars( initial_cwd.to_string_lossy().to_string(), ); } + if !env_vars.contains_key(crate::npm::NPM_CONFIG_USER_AGENT_ENV_VAR) { + env_vars.insert( + crate::npm::NPM_CONFIG_USER_AGENT_ENV_VAR.into(), + crate::npm::get_npm_config_user_agent(), + ); + } if let Some(node_modules_dir) = node_modules_dir { prepend_to_path( &mut env_vars, @@ -127,7 +210,7 @@ impl ShellCommand for NpmCommand { mut context: ShellCommandContext, ) -> LocalBoxFuture<'static, ExecuteResult> { if context.args.first().map(|s| s.as_str()) == Some("run") - && context.args.len() > 2 + && context.args.len() >= 2 // for now, don't run any npm scripts that have a flag because // we don't handle stuff like `--workspaces` properly && !context.args.iter().any(|s| s.starts_with('-')) @@ -190,10 +273,12 @@ impl ShellCommand for NodeCommand { ) .execute(context); } + args.extend(["run", "-A"].into_iter().map(|s| s.to_string())); args.extend(context.args.iter().cloned()); let mut state = context.state; + state.apply_env_var(USE_PKG_JSON_HIDDEN_ENV_VAR_NAME, "1"); ExecutableCommand::new("deno".to_string(), std::env::current_exe().unwrap()) .execute(ShellCommandContext { @@ -398,20 +483,32 @@ fn resolve_execution_path_from_npx_shim( static SCRIPT_PATH_RE: Lazy = lazy_regex::lazy_regex!(r#""\$basedir\/([^"]+)" "\$@""#); - if text.starts_with("#!/usr/bin/env node") { - // launch this file itself because it's a JS file - Some(file_path) - } else { - // Search for... - // > "$basedir/../next/dist/bin/next" "$@" - // ...which is what it will look like on Windows - SCRIPT_PATH_RE - .captures(text) - .and_then(|c| c.get(1)) - .map(|relative_path| { - file_path.parent().unwrap().join(relative_path.as_str()) - }) + let maybe_first_line = { + let index = text.find("\n")?; + Some(&text[0..index]) + }; + + if let Some(first_line) = maybe_first_line { + // NOTE(bartlomieju): this is not perfect, but handle two most common scenarios + // where Node is run without any args. If there are args then we use `NodeCommand` + // struct. + if first_line == "#!/usr/bin/env node" + || first_line == "#!/usr/bin/env -S node" + { + // launch this file itself because it's a JS file + return Some(file_path); + } } + + // Search for... + // > "$basedir/../next/dist/bin/next" "$@" + // ...which is what it will look like on Windows + SCRIPT_PATH_RE + .captures(text) + .and_then(|c| c.get(1)) + .map(|relative_path| { + file_path.parent().unwrap().join(relative_path.as_str()) + }) } fn resolve_managed_npm_commands( @@ -479,6 +576,16 @@ mod test { let unix_shim = r#"#!/usr/bin/env node "use strict"; console.log('Hi!'); +"#; + let path = PathBuf::from("/node_modules/.bin/example"); + assert_eq!( + resolve_execution_path_from_npx_shim(path.clone(), unix_shim).unwrap(), + path + ); + // example shim on unix + let unix_shim = r#"#!/usr/bin/env -S node +"use strict"; +console.log('Hi!'); "#; let path = PathBuf::from("/node_modules/.bin/example"); assert_eq!( diff --git a/cli/tools/bench/mod.rs b/cli/tools/bench/mod.rs index be5d0ad0e1e026..1d49fa061d2012 100644 --- a/cli/tools/bench/mod.rs +++ b/cli/tools/bench/mod.rs @@ -193,7 +193,7 @@ async fn bench_specifier_inner( .await?; // We execute the main module as a side module so that import.meta.main is not set. - worker.execute_side_module_possibly_with_npm().await?; + worker.execute_side_module().await?; let mut worker = worker.into_main_worker(); @@ -486,6 +486,7 @@ pub async fn run_benchmarks_with_watch( ), move |flags, watcher_communicator, changed_paths| { let bench_flags = bench_flags.clone(); + watcher_communicator.show_path_changed(changed_paths.clone()); Ok(async move { let factory = CliFactory::from_flags_for_watcher( flags, diff --git a/cli/tools/check.rs b/cli/tools/check.rs index c22afbb9a7fb86..ad5c7c3ab17726 100644 --- a/cli/tools/check.rs +++ b/cli/tools/check.rs @@ -32,6 +32,7 @@ use crate::graph_util::ModuleGraphBuilder; use crate::npm::CliNpmResolver; use crate::tsc; use crate::tsc::Diagnostics; +use crate::tsc::TypeCheckingCjsTracker; use crate::util::extract; use crate::util::path::to_percent_decoded_str; @@ -51,6 +52,7 @@ pub async fn check( let specifiers_for_typecheck = if check_flags.doc || check_flags.doc_only { let file_fetcher = factory.file_fetcher()?; + let root_permissions = factory.root_permissions_container()?; let mut specifiers_for_typecheck = if check_flags.doc { specifiers.clone() @@ -59,7 +61,7 @@ pub async fn check( }; for s in specifiers { - let file = file_fetcher.fetch_bypass_permissions(&s).await?; + let file = file_fetcher.fetch(&s, root_permissions).await?; let snippet_files = extract::extract_snippet_files(file)?; for snippet_file in snippet_files { specifiers_for_typecheck.push(snippet_file.specifier.clone()); @@ -98,6 +100,7 @@ pub struct CheckOptions { pub struct TypeChecker { caches: Arc, + cjs_tracker: Arc, cli_options: Arc, module_graph_builder: Arc, node_resolver: Arc, @@ -107,6 +110,7 @@ pub struct TypeChecker { impl TypeChecker { pub fn new( caches: Arc, + cjs_tracker: Arc, cli_options: Arc, module_graph_builder: Arc, node_resolver: Arc, @@ -114,6 +118,7 @@ impl TypeChecker { ) -> Self { Self { caches, + cjs_tracker, cli_options, module_graph_builder, node_resolver, @@ -243,6 +248,7 @@ impl TypeChecker { graph: graph.clone(), hash_data, maybe_npm: Some(tsc::RequestNpmState { + cjs_tracker: self.cjs_tracker.clone(), node_resolver: self.node_resolver.clone(), npm_resolver: self.npm_resolver.clone(), }), @@ -345,7 +351,7 @@ fn get_check_hash( } } MediaType::Json - | MediaType::TsBuildInfo + | MediaType::Css | MediaType::SourceMap | MediaType::Wasm | MediaType::Unknown => continue, @@ -374,6 +380,11 @@ fn get_check_hash( hasher.write_str(module.specifier.as_str()); hasher.write_str(&module.source); } + Module::Wasm(module) => { + has_file_to_type_check = true; + hasher.write_str(module.specifier.as_str()); + hasher.write_str(&module.source_dts); + } Module::External(module) => { hasher.write_str(module.specifier.as_str()); } @@ -427,10 +438,11 @@ fn get_tsc_roots( } MediaType::Json | MediaType::Wasm - | MediaType::TsBuildInfo + | MediaType::Css | MediaType::SourceMap | MediaType::Unknown => None, }, + Module::Wasm(module) => Some((module.specifier.clone(), MediaType::Dmts)), Module::External(_) | Module::Node(_) | Module::Npm(_) @@ -535,7 +547,7 @@ fn has_ts_check(media_type: MediaType, file_text: &str) -> bool { | MediaType::Tsx | MediaType::Json | MediaType::Wasm - | MediaType::TsBuildInfo + | MediaType::Css | MediaType::SourceMap | MediaType::Unknown => false, } diff --git a/cli/tools/compile.rs b/cli/tools/compile.rs index b9620cfded8d99..4fa996368308ef 100644 --- a/cli/tools/compile.rs +++ b/cli/tools/compile.rs @@ -5,7 +5,9 @@ use crate::args::CompileFlags; use crate::args::Flags; use crate::factory::CliFactory; use crate::http_util::HttpClientProvider; +use crate::standalone::binary::StandaloneRelativeFileBaseUrl; use crate::standalone::is_standalone_binary; +use deno_ast::MediaType; use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; use deno_core::anyhow::Context; @@ -14,7 +16,6 @@ use deno_core::error::AnyError; use deno_core::resolve_url_or_path; use deno_graph::GraphKind; use deno_terminal::colors; -use eszip::EszipRelativeFileBaseUrl; use rand::Rng; use std::path::Path; use std::path::PathBuf; @@ -29,18 +30,14 @@ pub async fn compile( let factory = CliFactory::from_flags(flags); let cli_options = factory.cli_options()?; let module_graph_creator = factory.module_graph_creator().await?; - let parsed_source_cache = factory.parsed_source_cache(); let binary_writer = factory.create_compile_binary_writer().await?; let http_client = factory.http_client_provider(); - let module_specifier = cli_options.resolve_main_module()?; - let module_roots = { - let mut vec = Vec::with_capacity(compile_flags.include.len() + 1); - vec.push(module_specifier.clone()); - for side_module in &compile_flags.include { - vec.push(resolve_url_or_path(side_module, cli_options.initial_cwd())?); - } - vec - }; + let entrypoint = cli_options.resolve_main_module()?; + let (module_roots, include_files) = get_module_roots_and_include_files( + entrypoint, + &compile_flags, + cli_options.initial_cwd(), + )?; // this is not supported, so show a warning about it, but don't error in order // to allow someone to still run `deno compile` when this is in a deno.json @@ -70,7 +67,7 @@ pub async fn compile( let graph = if cli_options.type_check_mode().is_true() { // In this case, the previous graph creation did type checking, which will // create a module graph with types information in it. We don't want to - // store that in the eszip so create a code only module graph from scratch. + // store that in the binary so create a code only module graph from scratch. module_graph_creator .create_graph(GraphKind::CodeOnly, module_roots) .await? @@ -81,36 +78,24 @@ pub async fn compile( let ts_config_for_emit = cli_options .resolve_ts_config_for_emit(deno_config::deno_json::TsConfigType::Emit)?; check_warn_tsconfig(&ts_config_for_emit); - let (transpile_options, emit_options) = - crate::args::ts_config_to_transpile_and_emit_options( - ts_config_for_emit.ts_config, - )?; - let parser = parsed_source_cache.as_capturing_parser(); let root_dir_url = resolve_root_dir_from_specifiers( cli_options.workspace().root_dir(), - graph.specifiers().map(|(s, _)| s).chain( - cli_options - .node_modules_dir_path() - .and_then(|p| ModuleSpecifier::from_directory_path(p).ok()) - .iter(), - ), + graph + .specifiers() + .map(|(s, _)| s) + .chain( + cli_options + .node_modules_dir_path() + .and_then(|p| ModuleSpecifier::from_directory_path(p).ok()) + .iter(), + ) + .chain(include_files.iter()), ); log::debug!("Binary root dir: {}", root_dir_url); - let root_dir_url = EszipRelativeFileBaseUrl::new(&root_dir_url); - let eszip = eszip::EszipV2::from_graph(eszip::FromGraphOptions { - graph, - parser, - transpile_options, - emit_options, - // make all the modules relative to the root folder - relative_file_base: Some(root_dir_url), - npm_packages: None, - })?; - log::info!( "{} {} to {}", colors::green("Compile"), - module_specifier.to_string(), + entrypoint, output_path.display(), ); validate_output_path(&output_path)?; @@ -133,15 +118,18 @@ pub async fn compile( let write_result = binary_writer .write_bin( file, - eszip, - root_dir_url, - &module_specifier, + &graph, + StandaloneRelativeFileBaseUrl::from(&root_dir_url), + entrypoint, + &include_files, &compile_flags, - cli_options, ) .await .with_context(|| { - format!("Writing temporary file '{}'", temp_path.display()) + format!( + "Writing deno compile executable to temporary file '{}'", + temp_path.display() + ) }); // set it as executable @@ -226,6 +214,48 @@ fn validate_output_path(output_path: &Path) -> Result<(), AnyError> { Ok(()) } +fn get_module_roots_and_include_files( + entrypoint: &ModuleSpecifier, + compile_flags: &CompileFlags, + initial_cwd: &Path, +) -> Result<(Vec, Vec), AnyError> { + fn is_module_graph_module(url: &ModuleSpecifier) -> bool { + if url.scheme() != "file" { + return true; + } + let media_type = MediaType::from_specifier(url); + match media_type { + MediaType::JavaScript + | MediaType::Jsx + | MediaType::Mjs + | MediaType::Cjs + | MediaType::TypeScript + | MediaType::Mts + | MediaType::Cts + | MediaType::Dts + | MediaType::Dmts + | MediaType::Dcts + | MediaType::Tsx + | MediaType::Json + | MediaType::Wasm => true, + MediaType::Css | MediaType::SourceMap | MediaType::Unknown => false, + } + } + + let mut module_roots = Vec::with_capacity(compile_flags.include.len() + 1); + let mut include_files = Vec::with_capacity(compile_flags.include.len()); + module_roots.push(entrypoint.clone()); + for side_module in &compile_flags.include { + let url = resolve_url_or_path(side_module, initial_cwd)?; + if is_module_graph_module(&url) { + module_roots.push(url); + } else { + include_files.push(url); + } + } + Ok((module_roots, include_files)) +} + async fn resolve_compile_executable_output_path( http_client_provider: &HttpClientProvider, compile_flags: &CompileFlags, diff --git a/cli/tools/coverage/mod.rs b/cli/tools/coverage/mod.rs index 260c0c842477a7..2a554c13359ad2 100644 --- a/cli/tools/coverage/mod.rs +++ b/cli/tools/coverage/mod.rs @@ -6,12 +6,12 @@ use crate::args::FileFlags; use crate::args::Flags; use crate::cdp; use crate::factory::CliFactory; -use crate::npm::CliNpmResolver; use crate::tools::fmt::format_json; use crate::tools::test::is_supported_test_path; use crate::util::text_encoding::source_map_from_code; use deno_ast::MediaType; +use deno_ast::ModuleKind; use deno_ast::ModuleSpecifier; use deno_config::glob::FileCollector; use deno_config::glob::FilePatterns; @@ -25,6 +25,7 @@ use deno_core::serde_json; use deno_core::sourcemap::SourceMap; use deno_core::url::Url; use deno_core::LocalInspectorSession; +use node_resolver::InNpmPackageChecker; use regex::Regex; use std::fs; use std::fs::File; @@ -327,6 +328,7 @@ fn generate_coverage_report( coverage_report.found_lines = if let Some(source_map) = maybe_source_map.as_ref() { + let script_source_lines = script_source.lines().collect::>(); let mut found_lines = line_counts .iter() .enumerate() @@ -334,7 +336,23 @@ fn generate_coverage_report( // get all the mappings from this destination line to a different src line let mut results = source_map .tokens() - .filter(move |token| token.get_dst_line() as usize == index) + .filter(|token| { + let dst_line = token.get_dst_line() as usize; + dst_line == index && { + let dst_col = token.get_dst_col() as usize; + let content = script_source_lines + .get(dst_line) + .and_then(|line| { + line.get(dst_col..std::cmp::min(dst_col + 2, line.len())) + }) + .unwrap_or(""); + + !content.is_empty() + && content != "/*" + && content != "*/" + && content != "//" + } + }) .map(move |token| (token.get_src_line() as usize, *count)) .collect::>(); // only keep the results that point at different src lines @@ -444,7 +462,7 @@ fn filter_coverages( coverages: Vec, include: Vec, exclude: Vec, - npm_resolver: &dyn CliNpmResolver, + in_npm_pkg_checker: &dyn InNpmPackageChecker, ) -> Vec { let include: Vec = include.iter().map(|e| Regex::new(e).unwrap()).collect(); @@ -462,13 +480,13 @@ fn filter_coverages( .filter(|e| { let is_internal = e.url.starts_with("ext:") || e.url.ends_with("__anonymous__") - || e.url.ends_with("$deno$test.js") + || e.url.ends_with("$deno$test.mjs") || e.url.ends_with(".snap") || is_supported_test_path(Path::new(e.url.as_str())) || doc_test_re.is_match(e.url.as_str()) || Url::parse(&e.url) .ok() - .map(|url| npm_resolver.in_npm_package(&url)) + .map(|url| in_npm_pkg_checker.in_npm_package(&url)) .unwrap_or(false); let is_included = include.iter().any(|p| p.is_match(&e.url)); @@ -479,7 +497,7 @@ fn filter_coverages( .collect::>() } -pub async fn cover_files( +pub fn cover_files( flags: Arc, coverage_flags: CoverageFlags, ) -> Result<(), AnyError> { @@ -489,9 +507,10 @@ pub async fn cover_files( let factory = CliFactory::from_flags(flags); let cli_options = factory.cli_options()?; - let npm_resolver = factory.npm_resolver().await?; + let in_npm_pkg_checker = factory.in_npm_pkg_checker()?; let file_fetcher = factory.file_fetcher()?; let emitter = factory.emitter()?; + let cjs_tracker = factory.cjs_tracker()?; assert!(!coverage_flags.files.include.is_empty()); @@ -511,7 +530,7 @@ pub async fn cover_files( script_coverages, coverage_flags.include, coverage_flags.exclude, - npm_resolver.as_ref(), + in_npm_pkg_checker.as_ref(), ); if script_coverages.is_empty() { return Err(generic_error("No covered files included in the report")); @@ -568,16 +587,21 @@ pub async fn cover_files( let transpiled_code = match file.media_type { MediaType::JavaScript | MediaType::Unknown + | MediaType::Css + | MediaType::Wasm | MediaType::Cjs | MediaType::Mjs | MediaType::Json => None, - MediaType::Dts | MediaType::Dmts | MediaType::Dcts => Some(Vec::new()), + MediaType::Dts | MediaType::Dmts | MediaType::Dcts => Some(String::new()), MediaType::TypeScript | MediaType::Jsx | MediaType::Mts | MediaType::Cts | MediaType::Tsx => { - Some(match emitter.maybe_cached_emit(&file.specifier, &file.source) { + let module_kind = ModuleKind::from_is_cjs( + cjs_tracker.is_maybe_cjs(&file.specifier, file.media_type)?, + ); + Some(match emitter.maybe_cached_emit(&file.specifier, module_kind, &file.source) { Some(code) => code, None => { return Err(anyhow!( @@ -588,13 +612,12 @@ pub async fn cover_files( } }) } - MediaType::Wasm | MediaType::TsBuildInfo | MediaType::SourceMap => { + MediaType::SourceMap => { unreachable!() } }; let runtime_code: String = match transpiled_code { - Some(code) => String::from_utf8(code) - .with_context(|| format!("Failed decoding {}", file.specifier))?, + Some(code) => code, None => original_source.to_string(), }; diff --git a/cli/tools/doc.rs b/cli/tools/doc.rs index 0ba3b84fbba3b7..9a24e458ac9e9b 100644 --- a/cli/tools/doc.rs +++ b/cli/tools/doc.rs @@ -7,7 +7,9 @@ use crate::args::Flags; use crate::colors; use crate::display; use crate::factory::CliFactory; -use crate::graph_util::graph_exit_lock_errors; +use crate::graph_util::graph_exit_integrity_errors; +use crate::graph_util::graph_walk_errors; +use crate::graph_util::GraphWalkErrorsOptions; use crate::tsc::get_types_declaration_file_text; use crate::util::fs::collect_specifiers; use deno_ast::diagnostics::Diagnostic; @@ -19,10 +21,12 @@ use deno_core::error::AnyError; use deno_core::serde_json; use deno_doc as doc; use deno_doc::html::UrlResolveKind; +use deno_doc::html::UsageComposer; +use deno_doc::html::UsageComposerEntry; use deno_graph::source::NullFileSystem; +use deno_graph::EsParser; use deno_graph::GraphKind; use deno_graph::ModuleAnalyzer; -use deno_graph::ModuleParser; use deno_graph::ModuleSpecifier; use doc::html::ShortPath; use doc::DocDiagnostic; @@ -33,9 +37,12 @@ use std::sync::Arc; const JSON_SCHEMA_VERSION: u8 = 1; +const PRISM_CSS: &str = include_str!("./doc/prism.css"); +const PRISM_JS: &str = include_str!("./doc/prism.js"); + async fn generate_doc_nodes_for_builtin_types( doc_flags: DocFlags, - parser: &dyn ModuleParser, + parser: &dyn EsParser, analyzer: &dyn ModuleAnalyzer, ) -> Result>, AnyError> { let source_file_specifier = @@ -94,7 +101,7 @@ pub async fn doc( let module_info_cache = factory.module_info_cache()?; let parsed_source_cache = factory.parsed_source_cache(); let capturing_parser = parsed_source_cache.as_capturing_parser(); - let analyzer = module_info_cache.as_module_analyzer(parsed_source_cache); + let analyzer = module_info_cache.as_module_analyzer(); let doc_nodes_by_url = match doc_flags.source_files { DocSourceFileFlag::Builtin => { @@ -107,7 +114,7 @@ pub async fn doc( } DocSourceFileFlag::Paths(ref source_files) => { let module_graph_creator = factory.module_graph_creator().await?; - let maybe_lockfile = cli_options.maybe_lockfile(); + let fs = factory.fs(); let module_specifiers = collect_specifiers( FilePatterns { @@ -127,8 +134,18 @@ pub async fn doc( .create_graph(GraphKind::TypesOnly, module_specifiers.clone()) .await?; - if maybe_lockfile.is_some() { - graph_exit_lock_errors(&graph); + graph_exit_integrity_errors(&graph); + let errors = graph_walk_errors( + &graph, + fs, + &module_specifiers, + GraphWalkErrorsOptions { + check_js: false, + kind: GraphKind::TypesOnly, + }, + ); + for error in errors { + log::warn!("{} {}", colors::yellow("Warning"), error); } let doc_parser = doc::DocParser::new( @@ -183,7 +200,7 @@ pub async fn doc( kind_with_drilldown: deno_doc::html::DocNodeKindWithDrilldown::Other(node.kind()), inner: Rc::new(node), - drilldown_parent_kind: None, + drilldown_name: None, parent: None, }) .collect::>(), @@ -192,10 +209,14 @@ pub async fn doc( Default::default() }; + let mut main_entrypoint = None; + let rewrite_map = if let Some(config_file) = cli_options.start_dir.maybe_deno_json() { let config = config_file.to_exports_config()?; + main_entrypoint = config.get_resolved(".").ok().flatten(); + let rewrite_map = config .clone() .into_map() @@ -223,6 +244,7 @@ pub async fn doc( html_options, deno_ns, rewrite_map, + main_entrypoint, ) } else { let modules_len = doc_nodes_by_url.len(); @@ -250,7 +272,7 @@ pub async fn doc( } struct DocResolver { - deno_ns: std::collections::HashSet>, + deno_ns: std::collections::HashMap, Option>>, strip_trailing_html: bool, } @@ -274,7 +296,7 @@ impl deno_doc::html::HrefResolver for DocResolver { } fn resolve_global_symbol(&self, symbol: &[String]) -> Option { - if self.deno_ns.contains(symbol) { + if self.deno_ns.contains_key(symbol) { Some(format!( "https://deno.land/api@v{}?s={}", env!("CARGO_PKG_VERSION"), @@ -300,10 +322,6 @@ impl deno_doc::html::HrefResolver for DocResolver { None } - fn resolve_usage(&self, current_resolve: UrlResolveKind) -> Option { - current_resolve.get_file().map(|file| file.path.to_string()) - } - fn resolve_source(&self, location: &deno_doc::Location) -> Option { Some(location.filename.to_string()) } @@ -338,141 +356,47 @@ impl deno_doc::html::HrefResolver for DocResolver { } } -struct DenoDocResolver(bool); - -impl deno_doc::html::HrefResolver for DenoDocResolver { - fn resolve_path( - &self, - current: UrlResolveKind, - target: UrlResolveKind, - ) -> String { - let path = deno_doc::html::href_path_resolve(current, target); - if self.0 { - if let Some(path) = path - .strip_suffix("index.html") - .or_else(|| path.strip_suffix(".html")) - { - return path.to_owned(); - } - } - - path - } - - fn resolve_global_symbol(&self, _symbol: &[String]) -> Option { - None - } - - fn resolve_import_href( - &self, - _symbol: &[String], - _src: &str, - ) -> Option { - None - } - - fn resolve_usage(&self, _current_resolve: UrlResolveKind) -> Option { - None - } - - fn resolve_source(&self, _location: &deno_doc::Location) -> Option { - None - } - - fn resolve_external_jsdoc_module( - &self, - _module: &str, - _symbol: Option<&str>, - ) -> Option<(String, String)> { - None - } -} - -struct NodeDocResolver(bool); - -impl deno_doc::html::HrefResolver for NodeDocResolver { - fn resolve_path( - &self, - current: UrlResolveKind, - target: UrlResolveKind, - ) -> String { - let path = deno_doc::html::href_path_resolve(current, target); - if self.0 { - if let Some(path) = path - .strip_suffix("index.html") - .or_else(|| path.strip_suffix(".html")) - { - return path.to_owned(); - } - } - - path - } +struct DocComposer; - fn resolve_global_symbol(&self, _symbol: &[String]) -> Option { - None +impl UsageComposer for DocComposer { + fn is_single_mode(&self) -> bool { + true } - fn resolve_import_href( + fn compose( &self, - _symbol: &[String], - _src: &str, - ) -> Option { - None - } - - fn resolve_usage(&self, current_resolve: UrlResolveKind) -> Option { + current_resolve: UrlResolveKind, + usage_to_md: deno_doc::html::UsageToMd, + ) -> IndexMap { current_resolve .get_file() - .map(|file| format!("node:{}", file.path)) - } - - fn resolve_source(&self, _location: &deno_doc::Location) -> Option { - None - } - - fn resolve_external_jsdoc_module( - &self, - _module: &str, - _symbol: Option<&str>, - ) -> Option<(String, String)> { - None + .map(|current_file| { + IndexMap::from([( + UsageComposerEntry { + name: "".to_string(), + icon: None, + }, + usage_to_md(current_file.path.as_str(), None), + )]) + }) + .unwrap_or_default() } } fn generate_docs_directory( doc_nodes_by_url: IndexMap>, html_options: &DocHtmlFlag, - deno_ns: std::collections::HashSet>, + deno_ns: std::collections::HashMap, Option>>, rewrite_map: Option>, + main_entrypoint: Option, ) -> Result<(), AnyError> { let cwd = std::env::current_dir().context("Failed to get CWD")?; let output_dir_resolved = cwd.join(&html_options.output); - let internal_env = std::env::var("DENO_INTERNAL_HTML_DOCS").ok(); - - let href_resolver: Rc = if internal_env - .as_ref() - .is_some_and(|internal_html_docs| internal_html_docs == "node") - { - Rc::new(NodeDocResolver(html_options.strip_trailing_html)) - } else if internal_env - .as_ref() - .is_some_and(|internal_html_docs| internal_html_docs == "deno") - || deno_ns.is_empty() - { - Rc::new(DenoDocResolver(html_options.strip_trailing_html)) - } else { - Rc::new(DocResolver { - deno_ns, - strip_trailing_html: html_options.strip_trailing_html, - }) - }; - let category_docs = if let Some(category_docs_path) = &html_options.category_docs_path { let content = std::fs::read(category_docs_path)?; - Some(deno_core::serde_json::from_slice(&content)?) + Some(serde_json::from_slice(&content)?) } else { None }; @@ -481,7 +405,7 @@ fn generate_docs_directory( &html_options.symbol_redirect_map_path { let content = std::fs::read(symbol_redirect_map_path)?; - Some(deno_core::serde_json::from_slice(&content)?) + Some(serde_json::from_slice(&content)?) } else { None }; @@ -490,27 +414,42 @@ fn generate_docs_directory( &html_options.default_symbol_map_path { let content = std::fs::read(default_symbol_map_path)?; - Some(deno_core::serde_json::from_slice(&content)?) + Some(serde_json::from_slice(&content)?) } else { None }; let options = deno_doc::html::GenerateOptions { package_name: html_options.name.clone(), - main_entrypoint: None, + main_entrypoint, rewrite_map, - href_resolver, - usage_composer: None, - composable_output: false, + href_resolver: Rc::new(DocResolver { + deno_ns, + strip_trailing_html: html_options.strip_trailing_html, + }), + usage_composer: Rc::new(DocComposer), category_docs, - disable_search: internal_env.is_some(), + disable_search: false, symbol_redirect_map, default_symbol_map, + markdown_renderer: deno_doc::html::comrak::create_renderer( + None, None, None, + ), + markdown_stripper: Rc::new(deno_doc::html::comrak::strip), + head_inject: Some(Rc::new(|root| { + format!( + r#""#, + deno_doc::html::comrak::COMRAK_STYLESHEET_FILENAME + ) + })), }; - let files = deno_doc::html::generate(options, doc_nodes_by_url) + let mut files = deno_doc::html::generate(options, doc_nodes_by_url) .context("Failed to generate HTML documentation")?; + files.insert("prism.js".to_string(), PRISM_JS.to_string()); + files.insert("prism.css".to_string(), PRISM_CSS.to_string()); + let path = &output_dir_resolved; let _ = std::fs::remove_dir_all(path); std::fs::create_dir(path) diff --git a/cli/tools/doc/prism.css b/cli/tools/doc/prism.css new file mode 100644 index 00000000000000..afc2ef6ca98386 --- /dev/null +++ b/cli/tools/doc/prism.css @@ -0,0 +1,3 @@ +/* PrismJS 1.29.0 +https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript+bash+json+markdown+regex+rust+typescript */ +code[class*=language-],pre[class*=language-]{color:#000;background:0 0;text-shadow:0 1px #fff;font-family:Consolas,Monaco,'Andale Mono','Ubuntu Mono',monospace;font-size:1em;text-align:left;white-space:pre;word-spacing:normal;word-break:normal;word-wrap:normal;line-height:1.5;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-hyphens:none;-moz-hyphens:none;-ms-hyphens:none;hyphens:none}code[class*=language-] ::-moz-selection,code[class*=language-]::-moz-selection,pre[class*=language-] ::-moz-selection,pre[class*=language-]::-moz-selection{text-shadow:none;background:#b3d4fc}code[class*=language-] ::selection,code[class*=language-]::selection,pre[class*=language-] ::selection,pre[class*=language-]::selection{text-shadow:none;background:#b3d4fc}@media print{code[class*=language-],pre[class*=language-]{text-shadow:none}}pre[class*=language-]{overflow:auto}:not(pre)>code[class*=language-],pre[class*=language-]{background:#f5f2f0}:not(pre)>code[class*=language-]{padding:.1em;border-radius:.3em;white-space:normal}.token.cdata,.token.comment,.token.doctype,.token.prolog{color:#708090}.token.punctuation{color:#999}.token.namespace{opacity:.7}.token.boolean,.token.constant,.token.deleted,.token.number,.token.property,.token.symbol,.token.tag{color:#905}.token.attr-name,.token.builtin,.token.char,.token.inserted,.token.selector,.token.string{color:#690}.language-css .token.string,.style .token.string,.token.entity,.token.operator,.token.url{color:#9a6e3a;background:hsla(0,0%,100%,.5)}.token.atrule,.token.attr-value,.token.keyword{color:#07a}.token.class-name,.token.function{color:#dd4a68}.token.important,.token.regex,.token.variable{color:#e90}.token.bold,.token.important{font-weight:700}.token.italic{font-style:italic}.token.entity{cursor:help} diff --git a/cli/tools/doc/prism.js b/cli/tools/doc/prism.js new file mode 100644 index 00000000000000..23bf91958907f3 --- /dev/null +++ b/cli/tools/doc/prism.js @@ -0,0 +1,15 @@ +// MIT LICENSE +// Copyright (c) 2012 Lea Verou +/* PrismJS 1.29.0 +https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript+bash+json+markdown+regex+rust+typescript */ +var _self="undefined"!=typeof window?window:"undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?self:{},Prism=function(e){var n=/(?:^|\s)lang(?:uage)?-([\w-]+)(?=\s|$)/i,t=0,r={},a={manual:e.Prism&&e.Prism.manual,disableWorkerMessageHandler:e.Prism&&e.Prism.disableWorkerMessageHandler,util:{encode:function e(n){return n instanceof i?new i(n.type,e(n.content),n.alias):Array.isArray(n)?n.map(e):n.replace(/&/g,"&").replace(/=g.reach);A+=w.value.length,w=w.next){var E=w.value;if(n.length>e.length)return;if(!(E instanceof i)){var P,L=1;if(y){if(!(P=l(b,A,e,m))||P.index>=e.length)break;var S=P.index,O=P.index+P[0].length,j=A;for(j+=w.value.length;S>=j;)j+=(w=w.next).value.length;if(A=j-=w.value.length,w.value instanceof i)continue;for(var C=w;C!==n.tail&&(jg.reach&&(g.reach=W);var z=w.prev;if(_&&(z=u(n,z,_),A+=_.length),c(n,z,L),w=u(n,z,new i(f,p?a.tokenize(N,p):N,k,N)),M&&u(n,w,M),L>1){var I={cause:f+","+d,reach:W};o(e,n,t,w.prev,A,I),g&&I.reach>g.reach&&(g.reach=I.reach)}}}}}}function s(){var e={value:null,prev:null,next:null},n={value:null,prev:e,next:null};e.next=n,this.head=e,this.tail=n,this.length=0}function u(e,n,t){var r=n.next,a={value:t,prev:n,next:r};return n.next=a,r.prev=a,e.length++,a}function c(e,n,t){for(var r=n.next,a=0;a"+i.content+""},!e.document)return e.addEventListener?(a.disableWorkerMessageHandler||e.addEventListener("message",(function(n){var t=JSON.parse(n.data),r=t.language,i=t.code,l=t.immediateClose;e.postMessage(a.highlight(i,a.languages[r],r)),l&&e.close()}),!1),a):a;var g=a.util.currentScript();function f(){a.manual||a.highlightAll()}if(g&&(a.filename=g.src,g.hasAttribute("data-manual")&&(a.manual=!0)),!a.manual){var h=document.readyState;"loading"===h||"interactive"===h&&g&&g.defer?document.addEventListener("DOMContentLoaded",f):window.requestAnimationFrame?window.requestAnimationFrame(f):window.setTimeout(f,16)}return a}(_self);"undefined"!=typeof module&&module.exports&&(module.exports=Prism),"undefined"!=typeof global&&(global.Prism=Prism); +Prism.languages.markup={comment:{pattern://,greedy:!0},prolog:{pattern:/<\?[\s\S]+?\?>/,greedy:!0},doctype:{pattern:/"'[\]]|"[^"]*"|'[^']*')+(?:\[(?:[^<"'\]]|"[^"]*"|'[^']*'|<(?!!--)|)*\]\s*)?>/i,greedy:!0,inside:{"internal-subset":{pattern:/(^[^\[]*\[)[\s\S]+(?=\]>$)/,lookbehind:!0,greedy:!0,inside:null},string:{pattern:/"[^"]*"|'[^']*'/,greedy:!0},punctuation:/^$|[[\]]/,"doctype-tag":/^DOCTYPE/i,name:/[^\s<>'"]+/}},cdata:{pattern://i,greedy:!0},tag:{pattern:/<\/?(?!\d)[^\s>\/=$<%]+(?:\s(?:\s*[^\s>\/=]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+(?=[\s>]))|(?=[\s/>])))+)?\s*\/?>/,greedy:!0,inside:{tag:{pattern:/^<\/?[^\s>\/]+/,inside:{punctuation:/^<\/?/,namespace:/^[^\s>\/:]+:/}},"special-attr":[],"attr-value":{pattern:/=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+)/,inside:{punctuation:[{pattern:/^=/,alias:"attr-equals"},{pattern:/^(\s*)["']|["']$/,lookbehind:!0}]}},punctuation:/\/?>/,"attr-name":{pattern:/[^\s>\/]+/,inside:{namespace:/^[^\s>\/:]+:/}}}},entity:[{pattern:/&[\da-z]{1,8};/i,alias:"named-entity"},/&#x?[\da-f]{1,8};/i]},Prism.languages.markup.tag.inside["attr-value"].inside.entity=Prism.languages.markup.entity,Prism.languages.markup.doctype.inside["internal-subset"].inside=Prism.languages.markup,Prism.hooks.add("wrap",(function(a){"entity"===a.type&&(a.attributes.title=a.content.replace(/&/,"&"))})),Object.defineProperty(Prism.languages.markup.tag,"addInlined",{value:function(a,e){var s={};s["language-"+e]={pattern:/(^$)/i,lookbehind:!0,inside:Prism.languages[e]},s.cdata=/^$/i;var t={"included-cdata":{pattern://i,inside:s}};t["language-"+e]={pattern:/[\s\S]+/,inside:Prism.languages[e]};var n={};n[a]={pattern:RegExp("(<__[^>]*>)(?:))*\\]\\]>|(?!)".replace(/__/g,(function(){return a})),"i"),lookbehind:!0,greedy:!0,inside:t},Prism.languages.insertBefore("markup","cdata",n)}}),Object.defineProperty(Prism.languages.markup.tag,"addAttribute",{value:function(a,e){Prism.languages.markup.tag.inside["special-attr"].push({pattern:RegExp("(^|[\"'\\s])(?:"+a+")\\s*=\\s*(?:\"[^\"]*\"|'[^']*'|[^\\s'\">=]+(?=[\\s>]))","i"),lookbehind:!0,inside:{"attr-name":/^[^\s=]+/,"attr-value":{pattern:/=[\s\S]+/,inside:{value:{pattern:/(^=\s*(["']|(?!["'])))\S[\s\S]*(?=\2$)/,lookbehind:!0,alias:[e,"language-"+e],inside:Prism.languages[e]},punctuation:[{pattern:/^=/,alias:"attr-equals"},/"|'/]}}}})}}),Prism.languages.html=Prism.languages.markup,Prism.languages.mathml=Prism.languages.markup,Prism.languages.svg=Prism.languages.markup,Prism.languages.xml=Prism.languages.extend("markup",{}),Prism.languages.ssml=Prism.languages.xml,Prism.languages.atom=Prism.languages.xml,Prism.languages.rss=Prism.languages.xml; +!function(s){var e=/(?:"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"|'(?:\\(?:\r\n|[\s\S])|[^'\\\r\n])*')/;s.languages.css={comment:/\/\*[\s\S]*?\*\//,atrule:{pattern:RegExp("@[\\w-](?:[^;{\\s\"']|\\s+(?!\\s)|"+e.source+")*?(?:;|(?=\\s*\\{))"),inside:{rule:/^@[\w-]+/,"selector-function-argument":{pattern:/(\bselector\s*\(\s*(?![\s)]))(?:[^()\s]|\s+(?![\s)])|\((?:[^()]|\([^()]*\))*\))+(?=\s*\))/,lookbehind:!0,alias:"selector"},keyword:{pattern:/(^|[^\w-])(?:and|not|only|or)(?![\w-])/,lookbehind:!0}}},url:{pattern:RegExp("\\burl\\((?:"+e.source+"|(?:[^\\\\\r\n()\"']|\\\\[^])*)\\)","i"),greedy:!0,inside:{function:/^url/i,punctuation:/^\(|\)$/,string:{pattern:RegExp("^"+e.source+"$"),alias:"url"}}},selector:{pattern:RegExp("(^|[{}\\s])[^{}\\s](?:[^{};\"'\\s]|\\s+(?![\\s{])|"+e.source+")*(?=\\s*\\{)"),lookbehind:!0},string:{pattern:e,greedy:!0},property:{pattern:/(^|[^-\w\xA0-\uFFFF])(?!\s)[-_a-z\xA0-\uFFFF](?:(?!\s)[-\w\xA0-\uFFFF])*(?=\s*:)/i,lookbehind:!0},important:/!important\b/i,function:{pattern:/(^|[^-a-z0-9])[-a-z0-9]+(?=\()/i,lookbehind:!0},punctuation:/[(){};:,]/},s.languages.css.atrule.inside.rest=s.languages.css;var t=s.languages.markup;t&&(t.tag.addInlined("style","css"),t.tag.addAttribute("style","css"))}(Prism); +Prism.languages.clike={comment:[{pattern:/(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/,lookbehind:!0,greedy:!0},{pattern:/(^|[^\\:])\/\/.*/,lookbehind:!0,greedy:!0}],string:{pattern:/(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,greedy:!0},"class-name":{pattern:/(\b(?:class|extends|implements|instanceof|interface|new|trait)\s+|\bcatch\s+\()[\w.\\]+/i,lookbehind:!0,inside:{punctuation:/[.\\]/}},keyword:/\b(?:break|catch|continue|do|else|finally|for|function|if|in|instanceof|new|null|return|throw|try|while)\b/,boolean:/\b(?:false|true)\b/,function:/\b\w+(?=\()/,number:/\b0x[\da-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?/i,operator:/[<>]=?|[!=]=?=?|--?|\+\+?|&&?|\|\|?|[?*/~^%]/,punctuation:/[{}[\];(),.:]/}; +Prism.languages.javascript=Prism.languages.extend("clike",{"class-name":[Prism.languages.clike["class-name"],{pattern:/(^|[^$\w\xA0-\uFFFF])(?!\s)[_$A-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\.(?:constructor|prototype))/,lookbehind:!0}],keyword:[{pattern:/((?:^|\})\s*)catch\b/,lookbehind:!0},{pattern:/(^|[^.]|\.\.\.\s*)\b(?:as|assert(?=\s*\{)|async(?=\s*(?:function\b|\(|[$\w\xA0-\uFFFF]|$))|await|break|case|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally(?=\s*(?:\{|$))|for|from(?=\s*(?:['"]|$))|function|(?:get|set)(?=\s*(?:[#\[$\w\xA0-\uFFFF]|$))|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)\b/,lookbehind:!0}],function:/#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*(?:\.\s*(?:apply|bind|call)\s*)?\()/,number:{pattern:RegExp("(^|[^\\w$])(?:NaN|Infinity|0[bB][01]+(?:_[01]+)*n?|0[oO][0-7]+(?:_[0-7]+)*n?|0[xX][\\dA-Fa-f]+(?:_[\\dA-Fa-f]+)*n?|\\d+(?:_\\d+)*n|(?:\\d+(?:_\\d+)*(?:\\.(?:\\d+(?:_\\d+)*)?)?|\\.\\d+(?:_\\d+)*)(?:[Ee][+-]?\\d+(?:_\\d+)*)?)(?![\\w$])"),lookbehind:!0},operator:/--|\+\+|\*\*=?|=>|&&=?|\|\|=?|[!=]==|<<=?|>>>?=?|[-+*/%&|^!=<>]=?|\.{3}|\?\?=?|\?\.?|[~:]/}),Prism.languages.javascript["class-name"][0].pattern=/(\b(?:class|extends|implements|instanceof|interface|new)\s+)[\w.\\]+/,Prism.languages.insertBefore("javascript","keyword",{regex:{pattern:RegExp("((?:^|[^$\\w\\xA0-\\uFFFF.\"'\\])\\s]|\\b(?:return|yield))\\s*)/(?:(?:\\[(?:[^\\]\\\\\r\n]|\\\\.)*\\]|\\\\.|[^/\\\\\\[\r\n])+/[dgimyus]{0,7}|(?:\\[(?:[^[\\]\\\\\r\n]|\\\\.|\\[(?:[^[\\]\\\\\r\n]|\\\\.|\\[(?:[^[\\]\\\\\r\n]|\\\\.)*\\])*\\])*\\]|\\\\.|[^/\\\\\\[\r\n])+/[dgimyus]{0,7}v[dgimyus]{0,7})(?=(?:\\s|/\\*(?:[^*]|\\*(?!/))*\\*/)*(?:$|[\r\n,.;:})\\]]|//))"),lookbehind:!0,greedy:!0,inside:{"regex-source":{pattern:/^(\/)[\s\S]+(?=\/[a-z]*$)/,lookbehind:!0,alias:"language-regex",inside:Prism.languages.regex},"regex-delimiter":/^\/|\/$/,"regex-flags":/^[a-z]+$/}},"function-variable":{pattern:/#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*[=:]\s*(?:async\s*)?(?:\bfunction\b|(?:\((?:[^()]|\([^()]*\))*\)|(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)\s*=>))/,alias:"function"},parameter:[{pattern:/(function(?:\s+(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)?\s*\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\))/,lookbehind:!0,inside:Prism.languages.javascript},{pattern:/(^|[^$\w\xA0-\uFFFF])(?!\s)[_$a-z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*=>)/i,lookbehind:!0,inside:Prism.languages.javascript},{pattern:/(\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\)\s*=>)/,lookbehind:!0,inside:Prism.languages.javascript},{pattern:/((?:\b|\s|^)(?!(?:as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)(?![$\w\xA0-\uFFFF]))(?:(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*\s*)\(\s*|\]\s*\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\)\s*\{)/,lookbehind:!0,inside:Prism.languages.javascript}],constant:/\b[A-Z](?:[A-Z_]|\dx?)*\b/}),Prism.languages.insertBefore("javascript","string",{hashbang:{pattern:/^#!.*/,greedy:!0,alias:"comment"},"template-string":{pattern:/`(?:\\[\s\S]|\$\{(?:[^{}]|\{(?:[^{}]|\{[^}]*\})*\})+\}|(?!\$\{)[^\\`])*`/,greedy:!0,inside:{"template-punctuation":{pattern:/^`|`$/,alias:"string"},interpolation:{pattern:/((?:^|[^\\])(?:\\{2})*)\$\{(?:[^{}]|\{(?:[^{}]|\{[^}]*\})*\})+\}/,lookbehind:!0,inside:{"interpolation-punctuation":{pattern:/^\$\{|\}$/,alias:"punctuation"},rest:Prism.languages.javascript}},string:/[\s\S]+/}},"string-property":{pattern:/((?:^|[,{])[ \t]*)(["'])(?:\\(?:\r\n|[\s\S])|(?!\2)[^\\\r\n])*\2(?=\s*:)/m,lookbehind:!0,greedy:!0,alias:"property"}}),Prism.languages.insertBefore("javascript","operator",{"literal-property":{pattern:/((?:^|[,{])[ \t]*)(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*:)/m,lookbehind:!0,alias:"property"}}),Prism.languages.markup&&(Prism.languages.markup.tag.addInlined("script","javascript"),Prism.languages.markup.tag.addAttribute("on(?:abort|blur|change|click|composition(?:end|start|update)|dblclick|error|focus(?:in|out)?|key(?:down|up)|load|mouse(?:down|enter|leave|move|out|over|up)|reset|resize|scroll|select|slotchange|submit|unload|wheel)","javascript")),Prism.languages.js=Prism.languages.javascript; +!function(e){var t="\\b(?:BASH|BASHOPTS|BASH_ALIASES|BASH_ARGC|BASH_ARGV|BASH_CMDS|BASH_COMPLETION_COMPAT_DIR|BASH_LINENO|BASH_REMATCH|BASH_SOURCE|BASH_VERSINFO|BASH_VERSION|COLORTERM|COLUMNS|COMP_WORDBREAKS|DBUS_SESSION_BUS_ADDRESS|DEFAULTS_PATH|DESKTOP_SESSION|DIRSTACK|DISPLAY|EUID|GDMSESSION|GDM_LANG|GNOME_KEYRING_CONTROL|GNOME_KEYRING_PID|GPG_AGENT_INFO|GROUPS|HISTCONTROL|HISTFILE|HISTFILESIZE|HISTSIZE|HOME|HOSTNAME|HOSTTYPE|IFS|INSTANCE|JOB|LANG|LANGUAGE|LC_ADDRESS|LC_ALL|LC_IDENTIFICATION|LC_MEASUREMENT|LC_MONETARY|LC_NAME|LC_NUMERIC|LC_PAPER|LC_TELEPHONE|LC_TIME|LESSCLOSE|LESSOPEN|LINES|LOGNAME|LS_COLORS|MACHTYPE|MAILCHECK|MANDATORY_PATH|NO_AT_BRIDGE|OLDPWD|OPTERR|OPTIND|ORBIT_SOCKETDIR|OSTYPE|PAPERSIZE|PATH|PIPESTATUS|PPID|PS1|PS2|PS3|PS4|PWD|RANDOM|REPLY|SECONDS|SELINUX_INIT|SESSION|SESSIONTYPE|SESSION_MANAGER|SHELL|SHELLOPTS|SHLVL|SSH_AUTH_SOCK|TERM|UID|UPSTART_EVENTS|UPSTART_INSTANCE|UPSTART_JOB|UPSTART_SESSION|USER|WINDOWID|XAUTHORITY|XDG_CONFIG_DIRS|XDG_CURRENT_DESKTOP|XDG_DATA_DIRS|XDG_GREETER_DATA_DIR|XDG_MENU_PREFIX|XDG_RUNTIME_DIR|XDG_SEAT|XDG_SEAT_PATH|XDG_SESSION_DESKTOP|XDG_SESSION_ID|XDG_SESSION_PATH|XDG_SESSION_TYPE|XDG_VTNR|XMODIFIERS)\\b",a={pattern:/(^(["']?)\w+\2)[ \t]+\S.*/,lookbehind:!0,alias:"punctuation",inside:null},n={bash:a,environment:{pattern:RegExp("\\$"+t),alias:"constant"},variable:[{pattern:/\$?\(\([\s\S]+?\)\)/,greedy:!0,inside:{variable:[{pattern:/(^\$\(\([\s\S]+)\)\)/,lookbehind:!0},/^\$\(\(/],number:/\b0x[\dA-Fa-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:[Ee]-?\d+)?/,operator:/--|\+\+|\*\*=?|<<=?|>>=?|&&|\|\||[=!+\-*/%<>^&|]=?|[?~:]/,punctuation:/\(\(?|\)\)?|,|;/}},{pattern:/\$\((?:\([^)]+\)|[^()])+\)|`[^`]+`/,greedy:!0,inside:{variable:/^\$\(|^`|\)$|`$/}},{pattern:/\$\{[^}]+\}/,greedy:!0,inside:{operator:/:[-=?+]?|[!\/]|##?|%%?|\^\^?|,,?/,punctuation:/[\[\]]/,environment:{pattern:RegExp("(\\{)"+t),lookbehind:!0,alias:"constant"}}},/\$(?:\w+|[#?*!@$])/],entity:/\\(?:[abceEfnrtv\\"]|O?[0-7]{1,3}|U[0-9a-fA-F]{8}|u[0-9a-fA-F]{4}|x[0-9a-fA-F]{1,2})/};e.languages.bash={shebang:{pattern:/^#!\s*\/.*/,alias:"important"},comment:{pattern:/(^|[^"{\\$])#.*/,lookbehind:!0},"function-name":[{pattern:/(\bfunction\s+)[\w-]+(?=(?:\s*\(?:\s*\))?\s*\{)/,lookbehind:!0,alias:"function"},{pattern:/\b[\w-]+(?=\s*\(\s*\)\s*\{)/,alias:"function"}],"for-or-select":{pattern:/(\b(?:for|select)\s+)\w+(?=\s+in\s)/,alias:"variable",lookbehind:!0},"assign-left":{pattern:/(^|[\s;|&]|[<>]\()\w+(?:\.\w+)*(?=\+?=)/,inside:{environment:{pattern:RegExp("(^|[\\s;|&]|[<>]\\()"+t),lookbehind:!0,alias:"constant"}},alias:"variable",lookbehind:!0},parameter:{pattern:/(^|\s)-{1,2}(?:\w+:[+-]?)?\w+(?:\.\w+)*(?=[=\s]|$)/,alias:"variable",lookbehind:!0},string:[{pattern:/((?:^|[^<])<<-?\s*)(\w+)\s[\s\S]*?(?:\r?\n|\r)\2/,lookbehind:!0,greedy:!0,inside:n},{pattern:/((?:^|[^<])<<-?\s*)(["'])(\w+)\2\s[\s\S]*?(?:\r?\n|\r)\3/,lookbehind:!0,greedy:!0,inside:{bash:a}},{pattern:/(^|[^\\](?:\\\\)*)"(?:\\[\s\S]|\$\([^)]+\)|\$(?!\()|`[^`]+`|[^"\\`$])*"/,lookbehind:!0,greedy:!0,inside:n},{pattern:/(^|[^$\\])'[^']*'/,lookbehind:!0,greedy:!0},{pattern:/\$'(?:[^'\\]|\\[\s\S])*'/,greedy:!0,inside:{entity:n.entity}}],environment:{pattern:RegExp("\\$?"+t),alias:"constant"},variable:n.variable,function:{pattern:/(^|[\s;|&]|[<>]\()(?:add|apropos|apt|apt-cache|apt-get|aptitude|aspell|automysqlbackup|awk|basename|bash|bc|bconsole|bg|bzip2|cal|cargo|cat|cfdisk|chgrp|chkconfig|chmod|chown|chroot|cksum|clear|cmp|column|comm|composer|cp|cron|crontab|csplit|curl|cut|date|dc|dd|ddrescue|debootstrap|df|diff|diff3|dig|dir|dircolors|dirname|dirs|dmesg|docker|docker-compose|du|egrep|eject|env|ethtool|expand|expect|expr|fdformat|fdisk|fg|fgrep|file|find|fmt|fold|format|free|fsck|ftp|fuser|gawk|git|gparted|grep|groupadd|groupdel|groupmod|groups|grub-mkconfig|gzip|halt|head|hg|history|host|hostname|htop|iconv|id|ifconfig|ifdown|ifup|import|install|ip|java|jobs|join|kill|killall|less|link|ln|locate|logname|logrotate|look|lpc|lpr|lprint|lprintd|lprintq|lprm|ls|lsof|lynx|make|man|mc|mdadm|mkconfig|mkdir|mke2fs|mkfifo|mkfs|mkisofs|mknod|mkswap|mmv|more|most|mount|mtools|mtr|mutt|mv|nano|nc|netstat|nice|nl|node|nohup|notify-send|npm|nslookup|op|open|parted|passwd|paste|pathchk|ping|pkill|pnpm|podman|podman-compose|popd|pr|printcap|printenv|ps|pushd|pv|quota|quotacheck|quotactl|ram|rar|rcp|reboot|remsync|rename|renice|rev|rm|rmdir|rpm|rsync|scp|screen|sdiff|sed|sendmail|seq|service|sftp|sh|shellcheck|shuf|shutdown|sleep|slocate|sort|split|ssh|stat|strace|su|sudo|sum|suspend|swapon|sync|sysctl|tac|tail|tar|tee|time|timeout|top|touch|tr|traceroute|tsort|tty|umount|uname|unexpand|uniq|units|unrar|unshar|unzip|update-grub|uptime|useradd|userdel|usermod|users|uudecode|uuencode|v|vcpkg|vdir|vi|vim|virsh|vmstat|wait|watch|wc|wget|whereis|which|who|whoami|write|xargs|xdg-open|yarn|yes|zenity|zip|zsh|zypper)(?=$|[)\s;|&])/,lookbehind:!0},keyword:{pattern:/(^|[\s;|&]|[<>]\()(?:case|do|done|elif|else|esac|fi|for|function|if|in|select|then|until|while)(?=$|[)\s;|&])/,lookbehind:!0},builtin:{pattern:/(^|[\s;|&]|[<>]\()(?:\.|:|alias|bind|break|builtin|caller|cd|command|continue|declare|echo|enable|eval|exec|exit|export|getopts|hash|help|let|local|logout|mapfile|printf|pwd|read|readarray|readonly|return|set|shift|shopt|source|test|times|trap|type|typeset|ulimit|umask|unalias|unset)(?=$|[)\s;|&])/,lookbehind:!0,alias:"class-name"},boolean:{pattern:/(^|[\s;|&]|[<>]\()(?:false|true)(?=$|[)\s;|&])/,lookbehind:!0},"file-descriptor":{pattern:/\B&\d\b/,alias:"important"},operator:{pattern:/\d?<>|>\||\+=|=[=~]?|!=?|<<[<-]?|[&\d]?>>|\d[<>]&?|[<>][&=]?|&[>&]?|\|[&|]?/,inside:{"file-descriptor":{pattern:/^\d/,alias:"important"}}},punctuation:/\$?\(\(?|\)\)?|\.\.|[{}[\];\\]/,number:{pattern:/(^|\s)(?:[1-9]\d*|0)(?:[.,]\d+)?\b/,lookbehind:!0}},a.inside=e.languages.bash;for(var s=["comment","function-name","for-or-select","assign-left","parameter","string","environment","function","keyword","builtin","boolean","file-descriptor","operator","punctuation","number"],o=n.variable[1].inside,i=0;i/g,(function(){return"(?:\\\\.|[^\\\\\n\r]|(?:\n|\r\n?)(?![\r\n]))"})),RegExp("((?:^|[^\\\\])(?:\\\\{2})*)(?:"+n+")")}var t="(?:\\\\.|``(?:[^`\r\n]|`(?!`))+``|`[^`\r\n]+`|[^\\\\|\r\n`])+",a="\\|?__(?:\\|__)+\\|?(?:(?:\n|\r\n?)|(?![^]))".replace(/__/g,(function(){return t})),i="\\|?[ \t]*:?-{3,}:?[ \t]*(?:\\|[ \t]*:?-{3,}:?[ \t]*)+\\|?(?:\n|\r\n?)";n.languages.markdown=n.languages.extend("markup",{}),n.languages.insertBefore("markdown","prolog",{"front-matter-block":{pattern:/(^(?:\s*[\r\n])?)---(?!.)[\s\S]*?[\r\n]---(?!.)/,lookbehind:!0,greedy:!0,inside:{punctuation:/^---|---$/,"front-matter":{pattern:/\S+(?:\s+\S+)*/,alias:["yaml","language-yaml"],inside:n.languages.yaml}}},blockquote:{pattern:/^>(?:[\t ]*>)*/m,alias:"punctuation"},table:{pattern:RegExp("^"+a+i+"(?:"+a+")*","m"),inside:{"table-data-rows":{pattern:RegExp("^("+a+i+")(?:"+a+")*$"),lookbehind:!0,inside:{"table-data":{pattern:RegExp(t),inside:n.languages.markdown},punctuation:/\|/}},"table-line":{pattern:RegExp("^("+a+")"+i+"$"),lookbehind:!0,inside:{punctuation:/\||:?-{3,}:?/}},"table-header-row":{pattern:RegExp("^"+a+"$"),inside:{"table-header":{pattern:RegExp(t),alias:"important",inside:n.languages.markdown},punctuation:/\|/}}}},code:[{pattern:/((?:^|\n)[ \t]*\n|(?:^|\r\n?)[ \t]*\r\n?)(?: {4}|\t).+(?:(?:\n|\r\n?)(?: {4}|\t).+)*/,lookbehind:!0,alias:"keyword"},{pattern:/^```[\s\S]*?^```$/m,greedy:!0,inside:{"code-block":{pattern:/^(```.*(?:\n|\r\n?))[\s\S]+?(?=(?:\n|\r\n?)^```$)/m,lookbehind:!0},"code-language":{pattern:/^(```).+/,lookbehind:!0},punctuation:/```/}}],title:[{pattern:/\S.*(?:\n|\r\n?)(?:==+|--+)(?=[ \t]*$)/m,alias:"important",inside:{punctuation:/==+$|--+$/}},{pattern:/(^\s*)#.+/m,lookbehind:!0,alias:"important",inside:{punctuation:/^#+|#+$/}}],hr:{pattern:/(^\s*)([*-])(?:[\t ]*\2){2,}(?=\s*$)/m,lookbehind:!0,alias:"punctuation"},list:{pattern:/(^\s*)(?:[*+-]|\d+\.)(?=[\t ].)/m,lookbehind:!0,alias:"punctuation"},"url-reference":{pattern:/!?\[[^\]]+\]:[\t ]+(?:\S+|<(?:\\.|[^>\\])+>)(?:[\t ]+(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\)))?/,inside:{variable:{pattern:/^(!?\[)[^\]]+/,lookbehind:!0},string:/(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\))$/,punctuation:/^[\[\]!:]|[<>]/},alias:"url"},bold:{pattern:e("\\b__(?:(?!_)|_(?:(?!_))+_)+__\\b|\\*\\*(?:(?!\\*)|\\*(?:(?!\\*))+\\*)+\\*\\*"),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^..)[\s\S]+(?=..$)/,lookbehind:!0,inside:{}},punctuation:/\*\*|__/}},italic:{pattern:e("\\b_(?:(?!_)|__(?:(?!_))+__)+_\\b|\\*(?:(?!\\*)|\\*\\*(?:(?!\\*))+\\*\\*)+\\*"),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^.)[\s\S]+(?=.$)/,lookbehind:!0,inside:{}},punctuation:/[*_]/}},strike:{pattern:e("(~~?)(?:(?!~))+\\2"),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^~~?)[\s\S]+(?=\1$)/,lookbehind:!0,inside:{}},punctuation:/~~?/}},"code-snippet":{pattern:/(^|[^\\`])(?:``[^`\r\n]+(?:`[^`\r\n]+)*``(?!`)|`[^`\r\n]+`(?!`))/,lookbehind:!0,greedy:!0,alias:["code","keyword"]},url:{pattern:e('!?\\[(?:(?!\\]))+\\](?:\\([^\\s)]+(?:[\t ]+"(?:\\\\.|[^"\\\\])*")?\\)|[ \t]?\\[(?:(?!\\]))+\\])'),lookbehind:!0,greedy:!0,inside:{operator:/^!/,content:{pattern:/(^\[)[^\]]+(?=\])/,lookbehind:!0,inside:{}},variable:{pattern:/(^\][ \t]?\[)[^\]]+(?=\]$)/,lookbehind:!0},url:{pattern:/(^\]\()[^\s)]+/,lookbehind:!0},string:{pattern:/(^[ \t]+)"(?:\\.|[^"\\])*"(?=\)$)/,lookbehind:!0}}}}),["url","bold","italic","strike"].forEach((function(e){["url","bold","italic","strike","code-snippet"].forEach((function(t){e!==t&&(n.languages.markdown[e].inside.content.inside[t]=n.languages.markdown[t])}))})),n.hooks.add("after-tokenize",(function(n){"markdown"!==n.language&&"md"!==n.language||function n(e){if(e&&"string"!=typeof e)for(var t=0,a=e.length;t",quot:'"'},l=String.fromCodePoint||String.fromCharCode;n.languages.md=n.languages.markdown}(Prism); +!function(a){var e={pattern:/\\[\\(){}[\]^$+*?|.]/,alias:"escape"},n=/\\(?:x[\da-fA-F]{2}|u[\da-fA-F]{4}|u\{[\da-fA-F]+\}|0[0-7]{0,2}|[123][0-7]{2}|c[a-zA-Z]|.)/,t="(?:[^\\\\-]|"+n.source+")",s=RegExp(t+"-"+t),i={pattern:/(<|')[^<>']+(?=[>']$)/,lookbehind:!0,alias:"variable"};a.languages.regex={"char-class":{pattern:/((?:^|[^\\])(?:\\\\)*)\[(?:[^\\\]]|\\[\s\S])*\]/,lookbehind:!0,inside:{"char-class-negation":{pattern:/(^\[)\^/,lookbehind:!0,alias:"operator"},"char-class-punctuation":{pattern:/^\[|\]$/,alias:"punctuation"},range:{pattern:s,inside:{escape:n,"range-punctuation":{pattern:/-/,alias:"operator"}}},"special-escape":e,"char-set":{pattern:/\\[wsd]|\\p\{[^{}]+\}/i,alias:"class-name"},escape:n}},"special-escape":e,"char-set":{pattern:/\.|\\[wsd]|\\p\{[^{}]+\}/i,alias:"class-name"},backreference:[{pattern:/\\(?![123][0-7]{2})[1-9]/,alias:"keyword"},{pattern:/\\k<[^<>']+>/,alias:"keyword",inside:{"group-name":i}}],anchor:{pattern:/[$^]|\\[ABbGZz]/,alias:"function"},escape:n,group:[{pattern:/\((?:\?(?:<[^<>']+>|'[^<>']+'|[>:]|)*\\*/",t=0;t<2;t++)a=a.replace(//g,(function(){return a}));a=a.replace(//g,(function(){return"[^\\s\\S]"})),e.languages.rust={comment:[{pattern:RegExp("(^|[^\\\\])"+a),lookbehind:!0,greedy:!0},{pattern:/(^|[^\\:])\/\/.*/,lookbehind:!0,greedy:!0}],string:{pattern:/b?"(?:\\[\s\S]|[^\\"])*"|b?r(#*)"(?:[^"]|"(?!\1))*"\1/,greedy:!0},char:{pattern:/b?'(?:\\(?:x[0-7][\da-fA-F]|u\{(?:[\da-fA-F]_*){1,6}\}|.)|[^\\\r\n\t'])'/,greedy:!0},attribute:{pattern:/#!?\[(?:[^\[\]"]|"(?:\\[\s\S]|[^\\"])*")*\]/,greedy:!0,alias:"attr-name",inside:{string:null}},"closure-params":{pattern:/([=(,:]\s*|\bmove\s*)\|[^|]*\||\|[^|]*\|(?=\s*(?:\{|->))/,lookbehind:!0,greedy:!0,inside:{"closure-punctuation":{pattern:/^\||\|$/,alias:"punctuation"},rest:null}},"lifetime-annotation":{pattern:/'\w+/,alias:"symbol"},"fragment-specifier":{pattern:/(\$\w+:)[a-z]+/,lookbehind:!0,alias:"punctuation"},variable:/\$\w+/,"function-definition":{pattern:/(\bfn\s+)\w+/,lookbehind:!0,alias:"function"},"type-definition":{pattern:/(\b(?:enum|struct|trait|type|union)\s+)\w+/,lookbehind:!0,alias:"class-name"},"module-declaration":[{pattern:/(\b(?:crate|mod)\s+)[a-z][a-z_\d]*/,lookbehind:!0,alias:"namespace"},{pattern:/(\b(?:crate|self|super)\s*)::\s*[a-z][a-z_\d]*\b(?:\s*::(?:\s*[a-z][a-z_\d]*\s*::)*)?/,lookbehind:!0,alias:"namespace",inside:{punctuation:/::/}}],keyword:[/\b(?:Self|abstract|as|async|await|become|box|break|const|continue|crate|do|dyn|else|enum|extern|final|fn|for|if|impl|in|let|loop|macro|match|mod|move|mut|override|priv|pub|ref|return|self|static|struct|super|trait|try|type|typeof|union|unsafe|unsized|use|virtual|where|while|yield)\b/,/\b(?:bool|char|f(?:32|64)|[ui](?:8|16|32|64|128|size)|str)\b/],function:/\b[a-z_]\w*(?=\s*(?:::\s*<|\())/,macro:{pattern:/\b\w+!/,alias:"property"},constant:/\b[A-Z_][A-Z_\d]+\b/,"class-name":/\b[A-Z]\w*\b/,namespace:{pattern:/(?:\b[a-z][a-z_\d]*\s*::\s*)*\b[a-z][a-z_\d]*\s*::(?!\s*<)/,inside:{punctuation:/::/}},number:/\b(?:0x[\dA-Fa-f](?:_?[\dA-Fa-f])*|0o[0-7](?:_?[0-7])*|0b[01](?:_?[01])*|(?:(?:\d(?:_?\d)*)?\.)?\d(?:_?\d)*(?:[Ee][+-]?\d+)?)(?:_?(?:f32|f64|[iu](?:8|16|32|64|size)?))?\b/,boolean:/\b(?:false|true)\b/,punctuation:/->|\.\.=|\.{1,3}|::|[{}[\];(),:]/,operator:/[-+*\/%!^]=?|=[=>]?|&[&=]?|\|[|=]?|<>?=?|[@?]/},e.languages.rust["closure-params"].inside.rest=e.languages.rust,e.languages.rust.attribute.inside.string=e.languages.rust.string}(Prism); +!function(e){e.languages.typescript=e.languages.extend("javascript",{"class-name":{pattern:/(\b(?:class|extends|implements|instanceof|interface|new|type)\s+)(?!keyof\b)(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?:\s*<(?:[^<>]|<(?:[^<>]|<[^<>]*>)*>)*>)?/,lookbehind:!0,greedy:!0,inside:null},builtin:/\b(?:Array|Function|Promise|any|boolean|console|never|number|string|symbol|unknown)\b/}),e.languages.typescript.keyword.push(/\b(?:abstract|declare|is|keyof|readonly|require)\b/,/\b(?:asserts|infer|interface|module|namespace|type)\b(?=\s*(?:[{_$a-zA-Z\xA0-\uFFFF]|$))/,/\btype\b(?=\s*(?:[\{*]|$))/),delete e.languages.typescript.parameter,delete e.languages.typescript["literal-property"];var s=e.languages.extend("typescript",{});delete s["class-name"],e.languages.typescript["class-name"].inside=s,e.languages.insertBefore("typescript","function",{decorator:{pattern:/@[$\w\xA0-\uFFFF]+/,inside:{at:{pattern:/^@/,alias:"operator"},function:/^[\s\S]+/}},"generic-function":{pattern:/#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*\s*<(?:[^<>]|<(?:[^<>]|<[^<>]*>)*>)*>(?=\s*\()/,greedy:!0,inside:{function:/^#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*/,generic:{pattern:/<[\s\S]+/,alias:"class-name",inside:s}}}}),e.languages.ts=e.languages.typescript}(Prism); diff --git a/cli/tools/fmt.rs b/cli/tools/fmt.rs index a7f4fd5543d7a7..c2c2a6bb6bf3ca 100644 --- a/cli/tools/fmt.rs +++ b/cli/tools/fmt.rs @@ -83,6 +83,7 @@ pub async fn format( file_watcher::PrintConfig::new("Fmt", !watch_flags.no_clear_screen), move |flags, watcher_communicator, changed_paths| { let fmt_flags = fmt_flags.clone(); + watcher_communicator.show_path_changed(changed_paths.clone()); Ok(async move { let factory = CliFactory::from_flags(flags); let cli_options = factory.cli_options()?; @@ -227,6 +228,7 @@ fn collect_fmt_files( }) .ignore_git_folder() .ignore_node_modules() + .use_gitignore() .set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned)) .collect_file_patterns(&deno_config::fs::RealDenoConfigFs, files) } @@ -270,6 +272,7 @@ fn format_markdown( | "njk" | "yml" | "yaml" + | "sql" ) { // It's important to tell dprint proper file extension, otherwise // it might parse the file twice. @@ -299,6 +302,13 @@ fn format_markdown( } } "yml" | "yaml" => format_yaml(text, fmt_options), + "sql" => { + if unstable_options.sql { + format_sql(text, fmt_options) + } else { + Ok(None) + } + } _ => { let mut codeblock_config = get_resolved_typescript_config(fmt_options); @@ -353,6 +363,21 @@ fn format_yaml( file_text: &str, fmt_options: &FmtOptionsConfig, ) -> Result, AnyError> { + let ignore_file = file_text + .lines() + .take_while(|line| line.starts_with('#')) + .any(|line| { + line + .strip_prefix('#') + .unwrap() + .trim() + .starts_with("deno-fmt-ignore-file") + }); + + if ignore_file { + return Ok(None); + } + let formatted_str = pretty_yaml::format_text(file_text, &get_resolved_yaml_config(fmt_options)) .map_err(AnyError::from)?; @@ -437,25 +462,15 @@ pub fn format_html( ) .map_err(|error| match error { markup_fmt::FormatError::Syntax(error) => { - // TODO(bartlomieju): rework when better error support in `markup_fmt` lands fn inner( error: &markup_fmt::SyntaxError, file_path: &Path, ) -> Option { - let error_str = format!("{}", error); - let error_str = error_str.strip_prefix("syntax error '")?; - - let reason = error_str - .split("' at") - .collect::>() - .first() - .map(|s| s.to_string())?; - let url = Url::from_file_path(file_path).ok()?; let error_msg = format!( "Syntax error ({}) at {}:{}:{}\n", - reason, + error.kind, url.as_str(), error.line, error.column @@ -496,7 +511,52 @@ pub fn format_html( }) } -/// Formats a single TS, TSX, JS, JSX, JSONC, JSON, MD, or IPYNB file. +pub fn format_sql( + file_text: &str, + fmt_options: &FmtOptionsConfig, +) -> Result, AnyError> { + let ignore_file = file_text + .lines() + .take_while(|line| line.starts_with("--")) + .any(|line| { + line + .strip_prefix("--") + .unwrap() + .trim() + .starts_with("deno-fmt-ignore-file") + }); + + if ignore_file { + return Ok(None); + } + + let mut formatted_str = sqlformat::format( + file_text, + &sqlformat::QueryParams::None, + &sqlformat::FormatOptions { + ignore_case_convert: None, + indent: if fmt_options.use_tabs.unwrap_or_default() { + sqlformat::Indent::Tabs + } else { + sqlformat::Indent::Spaces(fmt_options.indent_width.unwrap_or(2)) + }, + // leave one blank line between queries. + lines_between_queries: 2, + uppercase: Some(true), + }, + ); + + // Add single new line to the end of file. + formatted_str.push('\n'); + + Ok(if formatted_str == file_text { + None + } else { + Some(formatted_str) + }) +} + +/// Formats a single TS, TSX, JS, JSX, JSONC, JSON, MD, IPYNB or SQL file. pub fn format_file( file_path: &Path, file_text: &str, @@ -531,6 +591,13 @@ pub fn format_file( format_file(file_path, &file_text, fmt_options, unstable_options, None) }, ), + "sql" => { + if unstable_options.sql { + format_sql(file_text, fmt_options) + } else { + Ok(None) + } + } _ => { let config = get_resolved_typescript_config(fmt_options); dprint_plugin_typescript::format_text( @@ -785,28 +852,26 @@ fn format_ensure_stable( return Ok(Some(current_text)); } Err(err) => { - panic!( + bail!( concat!( "Formatting succeeded initially, but failed when ensuring a ", "stable format. This indicates a bug in the formatter where ", "the text it produces is not syntactically correct. As a temporary ", - "workaround you can ignore this file ({}).\n\n{:#}" + "workaround you can ignore this file.\n\n{:#}" ), - file_path.display(), err, ) } } count += 1; if count == 5 { - panic!( + bail!( concat!( "Formatting not stable. Bailed after {} tries. This indicates a bug ", - "in the formatter where it formats the file ({}) differently each time. As a ", + "in the formatter where it formats the file differently each time. As a ", "temporary workaround you can ignore this file." ), count, - file_path.display(), ) } } @@ -988,6 +1053,7 @@ fn get_resolved_malva_config( single_line_top_level_declarations: false, selector_override_comment_directive: "deno-fmt-selector-override".into(), ignore_comment_directive: "deno-fmt-ignore".into(), + ignore_file_comment_directive: "deno-fmt-ignore-file".into(), }; FormatOptions { @@ -1026,7 +1092,7 @@ fn get_resolved_markup_fmt_config( max_attrs_per_line: None, prefer_attrs_single_line: false, html_normal_self_closing: None, - html_void_self_closing: Some(true), + html_void_self_closing: None, component_self_closing: None, svg_self_closing: None, mathml_self_closing: None, @@ -1046,6 +1112,7 @@ fn get_resolved_markup_fmt_config( svelte_directive_shorthand: Some(true), astro_attr_shorthand: Some(true), ignore_comment_directive: "deno-fmt-ignore".into(), + ignore_file_comment_directive: "deno-fmt-ignore-file".into(), }; FormatOptions { @@ -1202,12 +1269,15 @@ fn is_supported_ext_fmt(path: &Path) -> bool { | "yml" | "yaml" | "ipynb" + | "sql" ) }) } #[cfg(test)] mod test { + use test_util::assert_starts_with; + use super::*; #[test] @@ -1260,15 +1330,24 @@ mod test { assert!(is_supported_ext_fmt(Path::new("foo.yaml"))); assert!(is_supported_ext_fmt(Path::new("foo.YaML"))); assert!(is_supported_ext_fmt(Path::new("foo.ipynb"))); + assert!(is_supported_ext_fmt(Path::new("foo.sql"))); + assert!(is_supported_ext_fmt(Path::new("foo.Sql"))); + assert!(is_supported_ext_fmt(Path::new("foo.sQl"))); + assert!(is_supported_ext_fmt(Path::new("foo.sqL"))); + assert!(is_supported_ext_fmt(Path::new("foo.SQL"))); } #[test] - #[should_panic(expected = "Formatting not stable. Bailed after 5 tries.")] fn test_format_ensure_stable_unstable_format() { - format_ensure_stable(&PathBuf::from("mod.ts"), "1", |_, file_text| { - Ok(Some(format!("1{file_text}"))) - }) - .unwrap(); + let err = + format_ensure_stable(&PathBuf::from("mod.ts"), "1", |_, file_text| { + Ok(Some(format!("1{file_text}"))) + }) + .unwrap_err(); + assert_starts_with!( + err.to_string(), + "Formatting not stable. Bailed after 5 tries." + ); } #[test] @@ -1282,16 +1361,20 @@ mod test { } #[test] - #[should_panic(expected = "Formatting succeeded initially, but failed when")] fn test_format_ensure_stable_error_second() { - format_ensure_stable(&PathBuf::from("mod.ts"), "1", |_, file_text| { - if file_text == "1" { - Ok(Some("11".to_string())) - } else { - bail!("Error formatting.") - } - }) - .unwrap(); + let err = + format_ensure_stable(&PathBuf::from("mod.ts"), "1", |_, file_text| { + if file_text == "1" { + Ok(Some("11".to_string())) + } else { + bail!("Error formatting.") + } + }) + .unwrap_err(); + assert_starts_with!( + err.to_string(), + "Formatting succeeded initially, but failed when" + ); } #[test] diff --git a/cli/tools/info.rs b/cli/tools/info.rs index d78b83cbe3bd0b..f0cd37772d0e29 100644 --- a/cli/tools/info.rs +++ b/cli/tools/info.rs @@ -11,12 +11,14 @@ use deno_core::anyhow::bail; use deno_core::error::AnyError; use deno_core::resolve_url_or_path; use deno_core::serde_json; +use deno_core::url; use deno_graph::Dependency; use deno_graph::GraphKind; use deno_graph::Module; use deno_graph::ModuleError; use deno_graph::ModuleGraph; use deno_graph::Resolution; +use deno_npm::npm_rc::ResolvedNpmRc; use deno_npm::resolution::NpmResolutionSnapshot; use deno_npm::NpmPackageId; use deno_npm::NpmResolutionPackage; @@ -29,7 +31,7 @@ use crate::args::Flags; use crate::args::InfoFlags; use crate::display; use crate::factory::CliFactory; -use crate::graph_util::graph_exit_lock_errors; +use crate::graph_util::graph_exit_integrity_errors; use crate::npm::CliNpmResolver; use crate::npm::ManagedCliNpmResolver; use crate::util::checksum; @@ -47,20 +49,71 @@ pub async fn info( let module_graph_creator = factory.module_graph_creator().await?; let npm_resolver = factory.npm_resolver().await?; let maybe_lockfile = cli_options.maybe_lockfile(); - let resolver = factory.workspace_resolver().await?; - - let maybe_import_specifier = - if let Some(import_map) = resolver.maybe_import_map() { - if let Ok(imports_specifier) = - import_map.resolve(&specifier, import_map.base_url()) - { - Some(imports_specifier) - } else { - None + let resolver = factory.workspace_resolver().await?.clone(); + let npmrc = cli_options.npmrc(); + let node_resolver = factory.node_resolver().await?; + + let cwd_url = + url::Url::from_directory_path(cli_options.initial_cwd()).unwrap(); + + let maybe_import_specifier = if let Ok(resolved) = + resolver.resolve(&specifier, &cwd_url) + { + match resolved { + deno_config::workspace::MappedResolution::Normal { + specifier, .. } - } else { - None - }; + | deno_config::workspace::MappedResolution::ImportMap { + specifier, + .. + } + | deno_config::workspace::MappedResolution::WorkspaceJsrPackage { + specifier, + .. + } => Some(specifier), + deno_config::workspace::MappedResolution::WorkspaceNpmPackage { + target_pkg_json, + sub_path, + .. + } => Some(node_resolver.resolve_package_subpath_from_deno_module( + target_pkg_json.clone().dir_path(), + sub_path.as_deref(), + Some(&cwd_url), + node_resolver::ResolutionMode::Import, + node_resolver::NodeResolutionKind::Execution, + )?), + deno_config::workspace::MappedResolution::PackageJson { + alias, + sub_path, + dep_result, + .. + } => match dep_result.as_ref().map_err(|e| e.clone())? { + deno_package_json::PackageJsonDepValue::Workspace(version_req) => { + let pkg_folder = resolver + .resolve_workspace_pkg_json_folder_for_pkg_json_dep( + alias, + version_req, + )?; + Some(node_resolver.resolve_package_subpath_from_deno_module( + pkg_folder, + sub_path.as_deref(), + Some(&cwd_url), + node_resolver::ResolutionMode::Import, + node_resolver::NodeResolutionKind::Execution, + )?) + } + deno_package_json::PackageJsonDepValue::Req(req) => { + Some(ModuleSpecifier::parse(&format!( + "npm:{}{}", + req, + sub_path.map(|s| format!("/{}", s)).unwrap_or_default() + ))?) + } + }, + } + } else { + None + }; let specifier = match maybe_import_specifier { Some(specifier) => specifier, @@ -75,16 +128,21 @@ pub async fn info( // write out the lockfile if there is one if let Some(lockfile) = &maybe_lockfile { - graph_exit_lock_errors(&graph); + graph_exit_integrity_errors(&graph); lockfile.write_if_changed()?; } if info_flags.json { let mut json_graph = serde_json::json!(graph); if let Some(output) = json_graph.as_object_mut() { - output.insert("version".to_string(), JSON_SCHEMA_VERSION.into()); + output.shift_insert( + 0, + "version".to_string(), + JSON_SCHEMA_VERSION.into(), + ); } - add_npm_packages_to_json(&mut json_graph, npm_resolver.as_ref()); + + add_npm_packages_to_json(&mut json_graph, npm_resolver.as_ref(), npmrc); display::write_json_to_stdout(&json_graph)?; } else { let mut output = String::new(); @@ -116,6 +174,7 @@ fn print_cache_info( let registry_cache = dir.registries_folder_path(); let mut origin_dir = dir.origin_data_folder_path(); let deno_dir = dir.root_path_for_display().to_string(); + let web_cache_dir = crate::worker::get_cache_storage_dir(); if let Some(location) = &location { origin_dir = @@ -133,6 +192,7 @@ fn print_cache_info( "typescriptCache": typescript_cache, "registryCache": registry_cache, "originStorage": origin_dir, + "webCacheStorage": web_cache_dir, }); if location.is_some() { @@ -167,6 +227,11 @@ fn print_cache_info( colors::bold("Origin storage:"), origin_dir.display() ); + println!( + "{} {}", + colors::bold("Web cache storage:"), + web_cache_dir.display() + ); if location.is_some() { println!( "{} {}", @@ -181,6 +246,7 @@ fn print_cache_info( fn add_npm_packages_to_json( json: &mut serde_json::Value, npm_resolver: &dyn CliNpmResolver, + npmrc: &ResolvedNpmRc, ) { let Some(npm_resolver) = npm_resolver.as_managed() else { return; // does not include byonm to deno info's output @@ -191,65 +257,57 @@ fn add_npm_packages_to_json( let json = json.as_object_mut().unwrap(); let modules = json.get_mut("modules").and_then(|m| m.as_array_mut()); if let Some(modules) = modules { - if modules.len() == 1 - && modules[0].get("kind").and_then(|k| k.as_str()) == Some("npm") - { - // If there is only one module and it's "external", then that means - // someone provided an npm specifier as a cli argument. In this case, - // we want to show which npm package the cli argument resolved to. - let module = &mut modules[0]; - let maybe_package = module - .get("specifier") - .and_then(|k| k.as_str()) - .and_then(|specifier| NpmPackageNvReference::from_str(specifier).ok()) - .and_then(|package_ref| { - snapshot - .resolve_package_from_deno_module(package_ref.nv()) - .ok() - }); - if let Some(pkg) = maybe_package { - if let Some(module) = module.as_object_mut() { - module - .insert("npmPackage".to_string(), pkg.id.as_serialized().into()); - } - } - } else { - // Filter out npm package references from the modules and instead - // have them only listed as dependencies. This is done because various - // npm specifiers modules in the graph are really just unresolved - // references. So there could be listed multiple npm specifiers - // that would resolve to a single npm package. - for i in (0..modules.len()).rev() { - if matches!( - modules[i].get("kind").and_then(|k| k.as_str()), - Some("npm") | Some("external") - ) { - modules.remove(i); + for module in modules.iter_mut() { + if matches!(module.get("kind").and_then(|k| k.as_str()), Some("npm")) { + // If there is only one module and it's "external", then that means + // someone provided an npm specifier as a cli argument. In this case, + // we want to show which npm package the cli argument resolved to. + let maybe_package = module + .get("specifier") + .and_then(|k| k.as_str()) + .and_then(|specifier| NpmPackageNvReference::from_str(specifier).ok()) + .and_then(|package_ref| { + snapshot + .resolve_package_from_deno_module(package_ref.nv()) + .ok() + }); + if let Some(pkg) = maybe_package { + if let Some(module) = module.as_object_mut() { + module + .insert("npmPackage".to_string(), pkg.id.as_serialized().into()); + } } } - } - for module in modules.iter_mut() { let dependencies = module .get_mut("dependencies") .and_then(|d| d.as_array_mut()); if let Some(dependencies) = dependencies { - for dep in dependencies.iter_mut() { - if let serde_json::Value::Object(dep) = dep { - let specifier = dep.get("specifier").and_then(|s| s.as_str()); - if let Some(specifier) = specifier { - if let Ok(npm_ref) = NpmPackageReqReference::from_str(specifier) { - if let Ok(pkg) = - snapshot.resolve_pkg_from_pkg_req(npm_ref.req()) - { - dep.insert( - "npmPackage".to_string(), - pkg.id.as_serialized().into(), - ); - } + for dep in dependencies.iter_mut().flat_map(|d| d.as_object_mut()) { + if let Some(specifier) = dep.get("specifier").and_then(|s| s.as_str()) + { + if let Ok(npm_ref) = NpmPackageReqReference::from_str(specifier) { + if let Ok(pkg) = snapshot.resolve_pkg_from_pkg_req(npm_ref.req()) + { + dep.insert( + "npmPackage".to_string(), + pkg.id.as_serialized().into(), + ); } } } + + // don't show this in the output unless someone needs it + if let Some(code) = + dep.get_mut("code").and_then(|c| c.as_object_mut()) + { + code.remove("resolutionMode"); + } + if let Some(types) = + dep.get_mut("types").and_then(|c| c.as_object_mut()) + { + types.remove("resolutionMode"); + } } } } @@ -261,7 +319,7 @@ fn add_npm_packages_to_json( let mut json_packages = serde_json::Map::with_capacity(sorted_packages.len()); for pkg in sorted_packages { let mut kv = serde_json::Map::new(); - kv.insert("name".to_string(), pkg.id.nv.name.to_string().into()); + kv.insert("name".to_string(), pkg.id.nv.name.clone().into()); kv.insert("version".to_string(), pkg.id.nv.version.to_string().into()); let mut deps = pkg.dependencies.values().collect::>(); deps.sort(); @@ -270,6 +328,8 @@ fn add_npm_packages_to_json( .map(|id| serde_json::Value::String(id.as_serialized())) .collect::>(); kv.insert("dependencies".to_string(), deps.into()); + let registry_url = npmrc.get_registry_url(&pkg.id.nv.name); + kv.insert("registryUrl".to_string(), registry_url.to_string().into()); json_packages.insert(pkg.id.as_serialized(), kv.into()); } @@ -450,6 +510,7 @@ impl<'a> GraphDisplayContext<'a> { let maybe_cache_info = match root { Module::Js(module) => module.maybe_cache_info.as_ref(), Module::Json(module) => module.maybe_cache_info.as_ref(), + Module::Wasm(module) => module.maybe_cache_info.as_ref(), Module::Node(_) | Module::Npm(_) | Module::External(_) => None, }; if let Some(cache_info) = maybe_cache_info { @@ -472,6 +533,7 @@ impl<'a> GraphDisplayContext<'a> { let size = match m { Module::Js(module) => module.size(), Module::Json(module) => module.size(), + Module::Wasm(module) => module.size(), Module::Node(_) | Module::Npm(_) | Module::External(_) => 0, }; size as f64 @@ -534,7 +596,7 @@ impl<'a> GraphDisplayContext<'a> { fn build_module_info(&mut self, module: &Module, type_dep: bool) -> TreeNode { enum PackageOrSpecifier { - Package(NpmResolutionPackage), + Package(Box), Specifier(ModuleSpecifier), } @@ -542,7 +604,7 @@ impl<'a> GraphDisplayContext<'a> { let package_or_specifier = match module.npm() { Some(npm) => match self.npm_info.resolve_package(npm.nv_reference.nv()) { - Some(package) => Package(package.clone()), + Some(package) => Package(Box::new(package.clone())), None => Specifier(module.specifier().clone()), // should never happen }, None => Specifier(module.specifier().clone()), @@ -571,6 +633,7 @@ impl<'a> GraphDisplayContext<'a> { Specifier(_) => match module { Module::Js(module) => Some(module.size() as u64), Module::Json(module) => Some(module.size() as u64), + Module::Wasm(module) => Some(module.size() as u64), Module::Node(_) | Module::Npm(_) | Module::External(_) => None, }, }; @@ -584,8 +647,8 @@ impl<'a> GraphDisplayContext<'a> { Package(package) => { tree_node.children.extend(self.build_npm_deps(package)); } - Specifier(_) => { - if let Some(module) = module.js() { + Specifier(_) => match module { + Module::Js(module) => { if let Some(types_dep) = &module.maybe_types_dependency { if let Some(child) = self.build_resolved_info(&types_dep.dependency, true) @@ -597,7 +660,16 @@ impl<'a> GraphDisplayContext<'a> { tree_node.children.extend(self.build_dep_info(dep)); } } - } + Module::Wasm(module) => { + for dep in module.dependencies.values() { + tree_node.children.extend(self.build_dep_info(dep)); + } + } + Module::Json(_) + | Module::Npm(_) + | Module::Node(_) + | Module::External(_) => {} + }, } } tree_node @@ -644,10 +716,25 @@ impl<'a> GraphDisplayContext<'a> { ModuleError::InvalidTypeAssertion { .. } => { self.build_error_msg(specifier, "(invalid import attribute)") } - ModuleError::LoadingErr(_, _, _) => { - self.build_error_msg(specifier, "(loading error)") + ModuleError::LoadingErr(_, _, err) => { + use deno_graph::ModuleLoadError::*; + let message = match err { + HttpsChecksumIntegrity(_) => "(checksum integrity error)", + Decode(_) => "(loading decode error)", + Loader(err) => { + match deno_runtime::errors::get_error_class_name(err) { + Some("NotCapable") => "(not capable, requires --allow-import)", + _ => "(loading error)", + } + } + Jsr(_) => "(loading error)", + NodeUnknownBuiltinModule(_) => "(unknown node built-in error)", + Npm(_) => "(npm loading error)", + TooManyRedirects => "(too many redirects error)", + }; + self.build_error_msg(specifier, message.as_ref()) } - ModuleError::ParseErr(_, _) => { + ModuleError::ParseErr(_, _) | ModuleError::WasmParseErr(_, _) => { self.build_error_msg(specifier, "(parsing error)") } ModuleError::UnsupportedImportAttributeType { .. } => { diff --git a/cli/tools/init/mod.rs b/cli/tools/init/mod.rs index 2d6a894e13c14b..36bdbac2bc6cad 100644 --- a/cli/tools/init/mod.rs +++ b/cli/tools/init/mod.rs @@ -1,15 +1,29 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use crate::args::DenoSubcommand; +use crate::args::Flags; use crate::args::InitFlags; +use crate::args::PackagesAllowedScripts; +use crate::args::PermissionFlags; +use crate::args::RunFlags; use crate::colors; +use color_print::cformat; +use color_print::cstr; +use deno_config::deno_json::NodeModulesDirMode; use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::serde_json::json; +use deno_runtime::WorkerExecutionMode; use log::info; +use std::io::IsTerminal; use std::io::Write; use std::path::Path; -pub fn init_project(init_flags: InitFlags) -> Result<(), AnyError> { +pub async fn init_project(init_flags: InitFlags) -> Result { + if let Some(package) = &init_flags.package { + return init_npm(package, init_flags.package_args).await; + } + let cwd = std::env::current_dir().context("Can't read current working directory.")?; let dir = if let Some(dir) = &init_flags.dir { @@ -24,32 +38,29 @@ pub fn init_project(init_flags: InitFlags) -> Result<(), AnyError> { create_file( &dir, "main.ts", - r#"import { type Route, route, serveDir } from "@std/http"; + r#"import { serveDir } from "@std/http"; -const routes: Route[] = [ - { - pattern: new URLPattern({ pathname: "/" }), - handler: () => new Response("Home page"), - }, - { - pattern: new URLPattern({ pathname: "/users/:id" }), - handler: (_req, _info, params) => new Response(params?.pathname.groups.id), - }, - { - pattern: new URLPattern({ pathname: "/static/*" }), - handler: (req) => serveDir(req), - }, -]; - -function defaultHandler(_req: Request) { - return new Response("Not found", { status: 404 }); -} - -const handler = route(routes, defaultHandler); +const userPagePattern = new URLPattern({ pathname: "/users/:id" }); +const staticPathPattern = new URLPattern({ pathname: "/static/*" }); export default { fetch(req) { - return handler(req); + const url = new URL(req.url); + + if (url.pathname === "/") { + return new Response("Home page"); + } + + const userPageMatch = userPagePattern.exec(url); + if (userPageMatch) { + return new Response(userPageMatch.pathname.groups.id); + } + + if (staticPathPattern.test(url)) { + return serveDir(req); + } + + return new Response("Not found", { status: 404 }); }, } satisfies Deno.ServeDefaultExport; "#, @@ -238,7 +249,97 @@ Deno.test(function addTest() { info!(" {}", colors::gray("# Run the tests")); info!(" deno test"); } - Ok(()) + Ok(0) +} + +fn npm_name_to_create_package(name: &str) -> String { + let mut s = "npm:".to_string(); + + let mut scoped = false; + let mut create = false; + + for (i, ch) in name.char_indices() { + if i == 0 { + if ch == '@' { + scoped = true; + } else { + create = true; + s.push_str("create-"); + } + } else if scoped { + if ch == '/' { + scoped = false; + create = true; + s.push_str("/create-"); + continue; + } else if ch == '@' && !create { + scoped = false; + create = true; + s.push_str("/create@"); + continue; + } + } + + s.push(ch); + } + + if !create { + s.push_str("/create"); + } + + s +} + +async fn init_npm(name: &str, args: Vec) -> Result { + let script_name = npm_name_to_create_package(name); + + fn print_manual_usage(script_name: &str, args: &[String]) -> i32 { + log::info!("{}", cformat!("You can initialize project manually by running deno run {} {} and applying desired permissions.", script_name, args.join(" "))); + 1 + } + + if std::io::stdin().is_terminal() { + log::info!( + cstr!("⚠️ Do you fully trust {} package? Deno will invoke code from it with all permissions. Do you want to continue? [y/n]"), + script_name + ); + loop { + let _ = std::io::stdout().write(b"> ")?; + std::io::stdout().flush()?; + let mut answer = String::new(); + if std::io::stdin().read_line(&mut answer).is_ok() { + let answer = answer.trim().to_ascii_lowercase(); + if answer != "y" { + return Ok(print_manual_usage(&script_name, &args)); + } else { + break; + } + } + } + } else { + return Ok(print_manual_usage(&script_name, &args)); + } + + let new_flags = Flags { + permissions: PermissionFlags { + allow_all: true, + ..Default::default() + }, + allow_scripts: PackagesAllowedScripts::All, + argv: args, + node_modules_dir: Some(NodeModulesDirMode::Auto), + subcommand: DenoSubcommand::Run(RunFlags { + script: script_name, + ..Default::default() + }), + ..Default::default() + }; + crate::tools::run::run_script( + WorkerExecutionMode::Run, + new_flags.into(), + None, + ) + .await } fn create_json_file( @@ -273,3 +374,37 @@ fn create_file( Ok(()) } } + +#[cfg(test)] +mod test { + use crate::tools::init::npm_name_to_create_package; + + #[test] + fn npm_name_to_create_package_test() { + // See https://docs.npmjs.com/cli/v8/commands/npm-init#description + assert_eq!( + npm_name_to_create_package("foo"), + "npm:create-foo".to_string() + ); + assert_eq!( + npm_name_to_create_package("foo@1.0.0"), + "npm:create-foo@1.0.0".to_string() + ); + assert_eq!( + npm_name_to_create_package("@foo"), + "npm:@foo/create".to_string() + ); + assert_eq!( + npm_name_to_create_package("@foo@1.0.0"), + "npm:@foo/create@1.0.0".to_string() + ); + assert_eq!( + npm_name_to_create_package("@foo/bar"), + "npm:@foo/create-bar".to_string() + ); + assert_eq!( + npm_name_to_create_package("@foo/bar@1.0.0"), + "npm:@foo/create-bar@1.0.0".to_string() + ); + } +} diff --git a/cli/tools/installer.rs b/cli/tools/installer.rs index bc8769de1c87b1..df5981e6e7b85e 100644 --- a/cli/tools/installer.rs +++ b/cli/tools/installer.rs @@ -3,6 +3,7 @@ use crate::args::resolve_no_prompt; use crate::args::AddFlags; use crate::args::CaData; +use crate::args::CacheSetting; use crate::args::ConfigFlag; use crate::args::Flags; use crate::args::InstallFlags; @@ -13,8 +14,11 @@ use crate::args::TypeCheckMode; use crate::args::UninstallFlags; use crate::args::UninstallKind; use crate::factory::CliFactory; +use crate::file_fetcher::FileFetcher; use crate::graph_container::ModuleGraphContainer; use crate::http_util::HttpClientProvider; +use crate::jsr::JsrFetchResolver; +use crate::npm::NpmFetchResolver; use crate::util::fs::canonicalize_path_maybe_not_exists; use deno_core::anyhow::bail; @@ -298,6 +302,10 @@ async fn install_local( } InstallFlagsLocal::TopLevel => { let factory = CliFactory::from_flags(flags); + // surface any errors in the package.json + if let Some(npm_resolver) = factory.npm_resolver().await?.as_managed() { + npm_resolver.ensure_no_pkg_json_dep_errors()?; + } crate::tools::registry::cache_top_level_deps(&factory, None).await?; if let Some(lockfile) = factory.cli_options()?.maybe_lockfile() { @@ -350,12 +358,54 @@ async fn install_global( ) -> Result<(), AnyError> { // ensure the module is cached let factory = CliFactory::from_flags(flags.clone()); + + let cli_options = factory.cli_options()?; + let http_client = factory.http_client_provider(); + let deps_http_cache = factory.global_http_cache()?; + let mut deps_file_fetcher = FileFetcher::new( + deps_http_cache.clone(), + CacheSetting::ReloadAll, + true, + http_client.clone(), + Default::default(), + None, + ); + + let npmrc = factory.cli_options().unwrap().npmrc(); + + deps_file_fetcher.set_download_log_level(log::Level::Trace); + let deps_file_fetcher = Arc::new(deps_file_fetcher); + let jsr_resolver = Arc::new(JsrFetchResolver::new(deps_file_fetcher.clone())); + let npm_resolver = Arc::new(NpmFetchResolver::new( + deps_file_fetcher.clone(), + npmrc.clone(), + )); + + let entry_text = install_flags_global.module_url.as_str(); + if !cli_options.initial_cwd().join(entry_text).exists() { + // check for package requirement missing prefix + if let Ok(Err(package_req)) = + super::registry::AddRmPackageReq::parse(entry_text) + { + if jsr_resolver.req_to_nv(&package_req).await.is_some() { + bail!( + "{entry_text} is missing a prefix. Did you mean `{}`?", + crate::colors::yellow(format!("deno install -g jsr:{package_req}")) + ); + } else if npm_resolver.req_to_nv(&package_req).await.is_some() { + bail!( + "{entry_text} is missing a prefix. Did you mean `{}`?", + crate::colors::yellow(format!("deno install -g npm:{package_req}")) + ); + } + } + } + factory .main_module_graph_container() .await? .load_and_type_check_files(&[install_flags_global.module_url.clone()]) .await?; - let http_client = factory.http_client_provider(); // create the install shim create_install_shim(http_client, &flags, install_flags_global).await @@ -1392,6 +1442,7 @@ mod tests { .env_clear() // use the deno binary in the target directory .env("PATH", test_util::target_dir()) + .env("RUST_BACKTRACE", "1") .spawn() .unwrap() .wait() diff --git a/cli/tools/jupyter/install.rs b/cli/tools/jupyter/install.rs index b0ddc948d6747f..aeff89ccf4f04a 100644 --- a/cli/tools/jupyter/install.rs +++ b/cli/tools/jupyter/install.rs @@ -58,9 +58,9 @@ pub fn install() -> Result<(), AnyError> { let f = std::fs::File::create(kernel_json_path)?; serde_json::to_writer_pretty(f, &json_data)?; - install_icon(&user_data_dir, "logo-32x32.png", DENO_ICON_32)?; - install_icon(&user_data_dir, "logo-64x64.png", DENO_ICON_64)?; - install_icon(&user_data_dir, "logo-svg.svg", DENO_ICON_SVG)?; + install_icon(&kernel_dir, "logo-32x32.png", DENO_ICON_32)?; + install_icon(&kernel_dir, "logo-64x64.png", DENO_ICON_64)?; + install_icon(&kernel_dir, "logo-svg.svg", DENO_ICON_SVG)?; log::info!("✅ Deno kernelspec installed successfully."); Ok(()) diff --git a/cli/tools/jupyter/mod.rs b/cli/tools/jupyter/mod.rs index 71e947dddb8623..732f95c49f25f7 100644 --- a/cli/tools/jupyter/mod.rs +++ b/cli/tools/jupyter/mod.rs @@ -61,7 +61,7 @@ pub async fn kernel( let factory = CliFactory::from_flags(flags); let cli_options = factory.cli_options()?; let main_module = - resolve_url_or_path("./$deno$jupyter.ts", cli_options.initial_cwd()) + resolve_url_or_path("./$deno$jupyter.mts", cli_options.initial_cwd()) .unwrap(); // TODO(bartlomieju): should we run with all permissions? let permissions = @@ -357,56 +357,74 @@ pub struct JupyterReplSession { impl JupyterReplSession { pub async fn start(&mut self) { + let mut poll_worker = true; loop { - let Some(msg) = self.rx.recv().await else { - break; - }; - let resp = match msg { - JupyterReplRequest::LspCompletions { - line_text, - position, - } => JupyterReplResponse::LspCompletions( - self.lsp_completions(&line_text, position).await, - ), - JupyterReplRequest::JsGetProperties { object_id } => { - JupyterReplResponse::JsGetProperties( - self.get_properties(object_id).await, - ) + tokio::select! { + biased; + + maybe_message = self.rx.recv() => { + let Some(msg) = maybe_message else { + break; + }; + if self.handle_message(msg).await.is_err() { + break; + } + poll_worker = true; + }, + _ = self.repl_session.run_event_loop(), if poll_worker => { + poll_worker = false; } - JupyterReplRequest::JsEvaluate { expr } => { - JupyterReplResponse::JsEvaluate(self.evaluate(expr).await) - } - JupyterReplRequest::JsGlobalLexicalScopeNames => { - JupyterReplResponse::JsGlobalLexicalScopeNames( - self.global_lexical_scope_names().await, - ) - } - JupyterReplRequest::JsEvaluateLineWithObjectWrapping { line } => { - JupyterReplResponse::JsEvaluateLineWithObjectWrapping( - self.evaluate_line_with_object_wrapping(&line).await, - ) - } - JupyterReplRequest::JsCallFunctionOnArgs { - function_declaration, - args, - } => JupyterReplResponse::JsCallFunctionOnArgs( - self - .call_function_on_args(function_declaration, &args) - .await, - ), - JupyterReplRequest::JsCallFunctionOn { arg0, arg1 } => { - JupyterReplResponse::JsCallFunctionOn( - self.call_function_on(arg0, arg1).await, - ) - } - }; - - let Ok(()) = self.tx.send(resp) else { - break; - }; + } } } + async fn handle_message( + &mut self, + msg: JupyterReplRequest, + ) -> Result<(), AnyError> { + let resp = match msg { + JupyterReplRequest::LspCompletions { + line_text, + position, + } => JupyterReplResponse::LspCompletions( + self.lsp_completions(&line_text, position).await, + ), + JupyterReplRequest::JsGetProperties { object_id } => { + JupyterReplResponse::JsGetProperties( + self.get_properties(object_id).await, + ) + } + JupyterReplRequest::JsEvaluate { expr } => { + JupyterReplResponse::JsEvaluate(self.evaluate(expr).await) + } + JupyterReplRequest::JsGlobalLexicalScopeNames => { + JupyterReplResponse::JsGlobalLexicalScopeNames( + self.global_lexical_scope_names().await, + ) + } + JupyterReplRequest::JsEvaluateLineWithObjectWrapping { line } => { + JupyterReplResponse::JsEvaluateLineWithObjectWrapping( + self.evaluate_line_with_object_wrapping(&line).await, + ) + } + JupyterReplRequest::JsCallFunctionOnArgs { + function_declaration, + args, + } => JupyterReplResponse::JsCallFunctionOnArgs( + self + .call_function_on_args(function_declaration, &args) + .await, + ), + JupyterReplRequest::JsCallFunctionOn { arg0, arg1 } => { + JupyterReplResponse::JsCallFunctionOn( + self.call_function_on(arg0, arg1).await, + ) + } + }; + + self.tx.send(resp).map_err(|e| e.into()) + } + pub async fn lsp_completions( &mut self, line_text: &str, diff --git a/cli/tools/jupyter/resources/deno-logo-32x32.png b/cli/tools/jupyter/resources/deno-logo-32x32.png index 97871a02ee768c..d59f251a243018 100644 Binary files a/cli/tools/jupyter/resources/deno-logo-32x32.png and b/cli/tools/jupyter/resources/deno-logo-32x32.png differ diff --git a/cli/tools/jupyter/resources/deno-logo-64x64.png b/cli/tools/jupyter/resources/deno-logo-64x64.png index 1b9444ef6370d8..37e98abafdf5db 100644 Binary files a/cli/tools/jupyter/resources/deno-logo-64x64.png and b/cli/tools/jupyter/resources/deno-logo-64x64.png differ diff --git a/cli/tools/jupyter/resources/deno-logo-svg.svg b/cli/tools/jupyter/resources/deno-logo-svg.svg index d7bb9ef804f33d..fbc22cd910d694 100644 --- a/cli/tools/jupyter/resources/deno-logo-svg.svg +++ b/cli/tools/jupyter/resources/deno-logo-svg.svg @@ -1 +1,17 @@ - + + + + + + + + + + + diff --git a/cli/tools/jupyter/server.rs b/cli/tools/jupyter/server.rs index 0cd80f7ddd9c24..5680ed4c13b878 100644 --- a/cli/tools/jupyter/server.rs +++ b/cli/tools/jupyter/server.rs @@ -329,7 +329,12 @@ impl JupyterServer { }) .collect(); - (candidates, cursor_pos - prop_name.len()) + if prop_name.len() > cursor_pos { + // TODO(bartlomieju): most likely not correct, but better than panicking because of sub with overflow + (candidates, cursor_pos) + } else { + (candidates, cursor_pos - prop_name.len()) + } } else { // combine results of declarations and globalThis properties let mut candidates = get_expression_property_names( @@ -349,7 +354,12 @@ impl JupyterServer { candidates.sort(); candidates.dedup(); // make sure to sort first - (candidates, cursor_pos - expr.len()) + if expr.len() > cursor_pos { + // TODO(bartlomieju): most likely not correct, but better than panicking because of sub with overflow + (candidates, cursor_pos) + } else { + (candidates, cursor_pos - expr.len()) + } }; connection diff --git a/cli/tools/lint/mod.rs b/cli/tools/lint/mod.rs index e096b486ef2b2f..fcefb458746f03 100644 --- a/cli/tools/lint/mod.rs +++ b/cli/tools/lint/mod.rs @@ -63,7 +63,7 @@ pub use rules::LintRuleProvider; const JSON_SCHEMA_VERSION: u8 = 1; -static STDIN_FILE_NAME: &str = "$deno$stdin.ts"; +static STDIN_FILE_NAME: &str = "$deno$stdin.mts"; pub async fn lint( flags: Arc, @@ -80,6 +80,7 @@ pub async fn lint( file_watcher::PrintConfig::new("Lint", !watch_flags.no_clear_screen), move |flags, watcher_communicator, changed_paths| { let lint_flags = lint_flags.clone(); + watcher_communicator.show_path_changed(changed_paths.clone()); Ok(async move { let factory = CliFactory::from_flags(flags); let cli_options = factory.cli_options()?; @@ -191,7 +192,7 @@ pub async fn lint( linter.finish() }; if !success { - std::process::exit(1); + deno_runtime::exit(1); } } @@ -435,6 +436,7 @@ fn collect_lint_files( }) .ignore_git_folder() .ignore_node_modules() + .use_gitignore() .set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned)) .collect_file_patterns(&deno_config::fs::RealDenoConfigFs, files) } diff --git a/cli/tools/lint/reporters.rs b/cli/tools/lint/reporters.rs index bf80be9f20ed58..18bc1216a69d7e 100644 --- a/cli/tools/lint/reporters.rs +++ b/cli/tools/lint/reporters.rs @@ -175,6 +175,7 @@ struct JsonLintReporter { version: u8, diagnostics: Vec, errors: Vec, + checked_files: Vec, } impl JsonLintReporter { @@ -183,6 +184,7 @@ impl JsonLintReporter { version: JSON_SCHEMA_VERSION, diagnostics: Vec::new(), errors: Vec::new(), + checked_files: Vec::new(), } } } @@ -209,6 +211,17 @@ impl LintReporter for JsonLintReporter { code: d.code().to_string(), hint: d.hint().map(|h| h.to_string()), }); + + let file_path = d + .specifier + .to_file_path() + .unwrap() + .to_string_lossy() + .to_string(); + + if !self.checked_files.contains(&file_path) { + self.checked_files.push(file_path); + } } fn visit_error(&mut self, file_path: &str, err: &AnyError) { @@ -216,10 +229,15 @@ impl LintReporter for JsonLintReporter { file_path: file_path.to_string(), message: err.to_string(), }); + + if !self.checked_files.contains(&file_path.to_string()) { + self.checked_files.push(file_path.to_string()); + } } fn close(&mut self, _check_count: usize) { sort_diagnostics(&mut self.diagnostics); + self.checked_files.sort(); let json = serde_json::to_string_pretty(&self); #[allow(clippy::print_stdout)] { diff --git a/cli/tools/lint/rules/mod.rs b/cli/tools/lint/rules/mod.rs index 2669ffda15fd01..dd723ad1598642 100644 --- a/cli/tools/lint/rules/mod.rs +++ b/cli/tools/lint/rules/mod.rs @@ -14,7 +14,7 @@ use deno_graph::ModuleGraph; use deno_lint::diagnostic::LintDiagnostic; use deno_lint::rules::LintRule; -use crate::resolver::SloppyImportsResolver; +use crate::resolver::CliSloppyImportsResolver; mod no_sloppy_imports; mod no_slow_types; @@ -144,13 +144,13 @@ impl ConfiguredRules { } pub struct LintRuleProvider { - sloppy_imports_resolver: Option>, + sloppy_imports_resolver: Option>, workspace_resolver: Option>, } impl LintRuleProvider { pub fn new( - sloppy_imports_resolver: Option>, + sloppy_imports_resolver: Option>, workspace_resolver: Option>, ) -> Self { Self { diff --git a/cli/tools/lint/rules/no_sloppy_imports.rs b/cli/tools/lint/rules/no_sloppy_imports.rs index 4180be5be1dd54..1bf7eddf6eeb7d 100644 --- a/cli/tools/lint/rules/no_sloppy_imports.rs +++ b/cli/tools/lint/rules/no_sloppy_imports.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use deno_ast::SourceRange; use deno_config::workspace::WorkspaceResolver; use deno_core::anyhow::anyhow; -use deno_graph::source::ResolutionMode; +use deno_graph::source::ResolutionKind; use deno_graph::source::ResolveError; use deno_graph::Range; use deno_lint::diagnostic::LintDiagnosticDetails; @@ -16,24 +16,25 @@ use deno_lint::diagnostic::LintDiagnosticRange; use deno_lint::diagnostic::LintFix; use deno_lint::diagnostic::LintFixChange; use deno_lint::rules::LintRule; +use deno_resolver::sloppy_imports::SloppyImportsResolution; +use deno_resolver::sloppy_imports::SloppyImportsResolutionKind; use text_lines::LineAndColumnIndex; use crate::graph_util::CliJsrUrlProvider; -use crate::resolver::SloppyImportsResolution; -use crate::resolver::SloppyImportsResolver; +use crate::resolver::CliSloppyImportsResolver; use super::ExtendedLintRule; #[derive(Debug)] pub struct NoSloppyImportsRule { - sloppy_imports_resolver: Option>, + sloppy_imports_resolver: Option>, // None for making printing out the lint rules easy workspace_resolver: Option>, } impl NoSloppyImportsRule { pub fn new( - sloppy_imports_resolver: Option>, + sloppy_imports_resolver: Option>, workspace_resolver: Option>, ) -> Self { NoSloppyImportsRule { @@ -86,6 +87,7 @@ impl LintRule for NoSloppyImportsRule { captures: Default::default(), }; + // fill this and capture the sloppy imports in the resolver deno_graph::parse_module_from_ast(deno_graph::ParseModuleFromAstOptions { graph_kind: deno_graph::GraphKind::All, specifier: context.specifier().clone(), @@ -99,16 +101,16 @@ impl LintRule for NoSloppyImportsRule { maybe_npm_resolver: None, }); - for (range, sloppy_import) in resolver.captures.borrow_mut().drain() { + for (referrer, sloppy_import) in resolver.captures.borrow_mut().drain() { let start_range = context.text_info().loc_to_source_pos(LineAndColumnIndex { - line_index: range.start.line, - column_index: range.start.character, + line_index: referrer.range.start.line, + column_index: referrer.range.start.character, }); let end_range = context.text_info().loc_to_source_pos(LineAndColumnIndex { - line_index: range.end.line, - column_index: range.end.character, + line_index: referrer.range.end.line, + column_index: referrer.range.end.character, }); let source_range = SourceRange::new(start_range, end_range); context.add_diagnostic_details( @@ -172,7 +174,7 @@ impl LintRule for NoSloppyImportsRule { #[derive(Debug)] struct SloppyImportCaptureResolver<'a> { workspace_resolver: &'a WorkspaceResolver, - sloppy_imports_resolver: &'a SloppyImportsResolver, + sloppy_imports_resolver: &'a CliSloppyImportsResolver, captures: RefCell>, } @@ -181,7 +183,7 @@ impl<'a> deno_graph::source::Resolver for SloppyImportCaptureResolver<'a> { &self, specifier_text: &str, referrer_range: &Range, - mode: ResolutionMode, + resolution_kind: ResolutionKind, ) -> Result { let resolution = self .workspace_resolver @@ -194,7 +196,13 @@ impl<'a> deno_graph::source::Resolver for SloppyImportCaptureResolver<'a> { } | deno_config::workspace::MappedResolution::ImportMap { specifier, .. - } => match self.sloppy_imports_resolver.resolve(&specifier, mode) { + } => match self.sloppy_imports_resolver.resolve( + &specifier, + match resolution_kind { + ResolutionKind::Execution => SloppyImportsResolutionKind::Execution, + ResolutionKind::Types => SloppyImportsResolutionKind::Types, + }, + ) { Some(res) => { self .captures diff --git a/cli/tools/registry/diagnostics.rs b/cli/tools/registry/diagnostics.rs index c53a39683ed64a..ef38affc30051d 100644 --- a/cli/tools/registry/diagnostics.rs +++ b/cli/tools/registry/diagnostics.rs @@ -3,7 +3,6 @@ use std::borrow::Cow; use std::path::PathBuf; use std::sync::Arc; -use std::sync::Mutex; use deno_ast::diagnostics::Diagnostic; use deno_ast::diagnostics::DiagnosticLevel; @@ -21,6 +20,7 @@ use deno_ast::SourceRanged; use deno_ast::SourceTextInfo; use deno_core::anyhow::anyhow; use deno_core::error::AnyError; +use deno_core::parking_lot::Mutex; use deno_core::url::Url; use deno_graph::FastCheckDiagnostic; use deno_semver::Version; @@ -36,7 +36,7 @@ impl PublishDiagnosticsCollector { pub fn print_and_error(&self) -> Result<(), AnyError> { let mut errors = 0; let mut has_slow_types_errors = false; - let mut diagnostics = self.diagnostics.lock().unwrap().take(); + let mut diagnostics = self.diagnostics.lock().take(); diagnostics.sort_by_cached_key(|d| d.sorting_key()); @@ -75,8 +75,16 @@ impl PublishDiagnosticsCollector { } } + pub fn has_error(&self) -> bool { + self + .diagnostics + .lock() + .iter() + .any(|d| matches!(d.level(), DiagnosticLevel::Error)) + } + pub fn push(&self, diagnostic: PublishDiagnostic) { - self.diagnostics.lock().unwrap().push(diagnostic); + self.diagnostics.lock().push(diagnostic); } } @@ -226,8 +234,8 @@ impl Diagnostic for PublishDiagnostic { specifier: Cow::Borrowed(&referrer.specifier), text_info: Cow::Borrowed(text_info), source_pos: DiagnosticSourcePos::LineAndCol { - line: referrer.start.line, - column: referrer.start.character, + line: referrer.range.start.line, + column: referrer.range.start.character, }, } } @@ -292,7 +300,7 @@ impl Diagnostic for PublishDiagnostic { text_info: &'a SourceTextInfo, referrer: &'a deno_graph::Range, ) -> Option> { - if referrer.start.line == 0 && referrer.start.character == 0 { + if referrer.range.start.line == 0 && referrer.range.start.character == 0 { return None; // no range, probably a jsxImportSource import } @@ -302,12 +310,12 @@ impl Diagnostic for PublishDiagnostic { style: DiagnosticSnippetHighlightStyle::Error, range: DiagnosticSourceRange { start: DiagnosticSourcePos::LineAndCol { - line: referrer.start.line, - column: referrer.start.character, + line: referrer.range.start.line, + column: referrer.range.start.character, }, end: DiagnosticSourcePos::LineAndCol { - line: referrer.end.line, - column: referrer.end.character, + line: referrer.range.end.line, + column: referrer.range.end.character, }, }, description: Some("the specifier".into()), @@ -468,7 +476,7 @@ impl Diagnostic for PublishDiagnostic { InvalidExternalImport { imported, .. } => Cow::Owned(vec![ Cow::Owned(format!("the import was resolved to '{}'", imported)), Cow::Borrowed("this specifier is not allowed to be imported on jsr"), - Cow::Borrowed("jsr only supports importing `jsr:`, `npm:`, and `data:` specifiers"), + Cow::Borrowed("jsr only supports importing `jsr:`, `npm:`, `data:`, `bun:`, and `node:` specifiers"), ]), UnsupportedJsxTsx { .. } => Cow::Owned(vec![ Cow::Borrowed("follow https://github.com/jsr-io/jsr/issues/24 for updates"), diff --git a/cli/tools/registry/graph.rs b/cli/tools/registry/graph.rs index 184557e5df7797..21962d009e1626 100644 --- a/cli/tools/registry/graph.rs +++ b/cli/tools/registry/graph.rs @@ -47,7 +47,7 @@ impl GraphDiagnosticsCollector { resolution: &ResolutionResolved| { if visited.insert(resolution.specifier.clone()) { match resolution.specifier.scheme() { - "file" | "data" | "node" => {} + "file" | "data" | "node" | "bun" => {} "jsr" => { skip_specifiers.insert(resolution.specifier.clone()); diff --git a/cli/tools/registry/mod.rs b/cli/tools/registry/mod.rs index fbdcd9e779fc59..a866660f366670 100644 --- a/cli/tools/registry/mod.rs +++ b/cli/tools/registry/mod.rs @@ -12,6 +12,7 @@ use std::sync::Arc; use base64::prelude::BASE64_STANDARD; use base64::Engine; use deno_ast::ModuleSpecifier; +use deno_config::deno_json::ConfigFile; use deno_config::workspace::JsrPackageConfig; use deno_config::workspace::PackageJsonDepResolution; use deno_config::workspace::Workspace; @@ -43,7 +44,8 @@ use crate::cache::ParsedSourceCache; use crate::factory::CliFactory; use crate::graph_util::ModuleGraphCreator; use crate::http_util::HttpClient; -use crate::resolver::SloppyImportsResolver; +use crate::resolver::CliSloppyImportsResolver; +use crate::resolver::SloppyImportsCachedFs; use crate::tools::check::CheckOptions; use crate::tools::lint::collect_no_slow_type_diagnostics; use crate::tools::registry::diagnostics::PublishDiagnostic; @@ -66,8 +68,10 @@ use auth::get_auth_method; use auth::AuthMethod; pub use pm::add; pub use pm::cache_top_level_deps; +pub use pm::outdated; pub use pm::remove; pub use pm::AddCommandName; +pub use pm::AddRmPackageReq; use publish_order::PublishOrderGraph; use unfurl::SpecifierUnfurler; @@ -88,15 +92,15 @@ pub async fn publish( let cli_options = cli_factory.cli_options()?; let directory_path = cli_options.initial_cwd(); - let publish_configs = cli_options.start_dir.jsr_packages_for_publish(); + let mut publish_configs = cli_options.start_dir.jsr_packages_for_publish(); if publish_configs.is_empty() { match cli_options.start_dir.maybe_deno_json() { Some(deno_json) => { debug_assert!(!deno_json.is_package()); - bail!( - "Missing 'name', 'version' and 'exports' field in '{}'.", - deno_json.specifier - ); + if deno_json.json.name.is_none() { + bail!("Missing 'name' field in '{}'.", deno_json.specifier); + } + error_missing_exports_field(deno_json)?; } None => { bail!( @@ -106,9 +110,23 @@ pub async fn publish( } } } + + if let Some(version) = &publish_flags.set_version { + if publish_configs.len() > 1 { + bail!("Cannot use --set-version when publishing a workspace. Change your cwd to an individual package instead."); + } + if let Some(publish_config) = publish_configs.get_mut(0) { + let mut config_file = publish_config.config_file.as_ref().clone(); + config_file.json.version = Some(version.clone()); + publish_config.config_file = Arc::new(config_file); + } + } + let specifier_unfurler = Arc::new(SpecifierUnfurler::new( if cli_options.unstable_sloppy_imports() { - Some(SloppyImportsResolver::new(cli_factory.fs().clone())) + Some(CliSloppyImportsResolver::new(SloppyImportsCachedFs::new( + cli_factory.fs().clone(), + ))) } else { None }, @@ -341,13 +359,11 @@ impl PublishPreparer { bail!("Exiting due to DENO_INTERNAL_FAST_CHECK_OVERWRITE") } else { log::info!("Checking for slow types in the public API..."); - let mut any_pkg_had_diagnostics = false; for package in package_configs { let export_urls = package.config_file.resolve_export_value_urls()?; let diagnostics = collect_no_slow_type_diagnostics(&graph, &export_urls); if !diagnostics.is_empty() { - any_pkg_had_diagnostics = true; for diagnostic in diagnostics { diagnostics_collector .push(PublishDiagnostic::FastCheck(diagnostic)); @@ -355,7 +371,9 @@ impl PublishPreparer { } } - if any_pkg_had_diagnostics { + // skip type checking the slow type graph if there are any errors because + // errors like remote modules existing will cause type checking to crash + if diagnostics_collector.has_error() { Ok(Arc::new(graph)) } else { // fast check passed, type check the output as a temporary measure @@ -400,43 +418,15 @@ impl PublishPreparer { graph: Arc, diagnostics_collector: &PublishDiagnosticsCollector, ) -> Result, AnyError> { - static SUGGESTED_ENTRYPOINTS: [&str; 4] = - ["mod.ts", "mod.js", "index.ts", "index.js"]; - let deno_json = &package.config_file; let config_path = deno_json.specifier.to_file_path().unwrap(); let root_dir = config_path.parent().unwrap().to_path_buf(); - let Some(version) = deno_json.json.version.clone() else { - bail!("{} is missing 'version' field", deno_json.specifier); - }; - if deno_json.json.exports.is_none() { - let mut suggested_entrypoint = None; - - for entrypoint in SUGGESTED_ENTRYPOINTS { - if root_dir.join(entrypoint).exists() { - suggested_entrypoint = Some(entrypoint); - break; - } - } - - let exports_content = format!( - r#"{{ - "name": "{}", - "version": "{}", - "exports": "{}" -}}"#, - package.name, - version, - suggested_entrypoint.unwrap_or("") - ); - - bail!( - "You did not specify an entrypoint to \"{}\" package in {}. Add `exports` mapping in the configuration file, eg:\n{}", - package.name, - deno_json.specifier, - exports_content - ); - } + let version = deno_json.json.version.clone().ok_or_else(|| { + deno_core::anyhow::anyhow!( + "{} is missing 'version' field", + deno_json.specifier + ) + })?; let Some(name_no_at) = package.name.strip_prefix('@') else { bail!("Invalid package name, use '@/ format"); }; @@ -1103,9 +1093,9 @@ fn collect_excluded_module_diagnostics( let graph_specifiers = graph .modules() .filter_map(|m| match m { - deno_graph::Module::Js(_) | deno_graph::Module::Json(_) => { - Some(m.specifier()) - } + deno_graph::Module::Js(_) + | deno_graph::Module::Json(_) + | deno_graph::Module::Wasm(_) => Some(m.specifier()), deno_graph::Module::Npm(_) | deno_graph::Module::Node(_) | deno_graph::Module::External(_) => None, @@ -1268,6 +1258,36 @@ fn has_license_file<'a>( }) } +fn error_missing_exports_field(deno_json: &ConfigFile) -> Result<(), AnyError> { + static SUGGESTED_ENTRYPOINTS: [&str; 4] = + ["mod.ts", "mod.js", "index.ts", "index.js"]; + let mut suggested_entrypoint = None; + + for entrypoint in SUGGESTED_ENTRYPOINTS { + if deno_json.dir_path().join(entrypoint).exists() { + suggested_entrypoint = Some(entrypoint); + break; + } + } + + let exports_content = format!( + r#"{{ + "name": "{}", + "version": "{}", + "exports": "{}" +}}"#, + deno_json.json.name.as_deref().unwrap_or("@scope/name"), + deno_json.json.name.as_deref().unwrap_or("0.0.0"), + suggested_entrypoint.unwrap_or("") + ); + + bail!( + "You did not specify an entrypoint in {}. Add `exports` mapping in the configuration file, eg:\n{}", + deno_json.specifier, + exports_content + ); +} + #[allow(clippy::print_stderr)] fn ring_bell() { // ASCII code for the bell character. diff --git a/cli/tools/registry/pm.rs b/cli/tools/registry/pm.rs index c92710f4602b80..5718cd3ec116fc 100644 --- a/cli/tools/registry/pm.rs +++ b/cli/tools/registry/pm.rs @@ -1,32 +1,26 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -mod cache_deps; - -pub use cache_deps::cache_top_level_deps; -use deno_semver::jsr::JsrPackageReqReference; -use deno_semver::npm::NpmPackageReqReference; -use deno_semver::VersionReq; - -use std::borrow::Cow; +use std::path::Path; use std::path::PathBuf; use std::sync::Arc; -use deno_ast::TextChange; -use deno_config::deno_json::FmtOptionsConfig; -use deno_core::anyhow::anyhow; use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::futures::FutureExt; use deno_core::futures::StreamExt; -use deno_core::serde_json; -use deno_core::ModuleSpecifier; -use deno_runtime::deno_node; +use deno_path_util::url_to_file_path; +use deno_semver::jsr::JsrPackageReqReference; +use deno_semver::npm::NpmPackageReqReference; +use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; -use indexmap::IndexMap; -use jsonc_parser::ast::ObjectProp; -use jsonc_parser::ast::Value; -use yoke::Yoke; +use deno_semver::Version; +use deno_semver::VersionReq; +use deps::KeyPath; +use jsonc_parser::cst::CstObject; +use jsonc_parser::cst::CstObjectProp; +use jsonc_parser::cst::CstRootNode; +use jsonc_parser::json; use crate::args::AddFlags; use crate::args::CacheSetting; @@ -38,234 +32,206 @@ use crate::file_fetcher::FileFetcher; use crate::jsr::JsrFetchResolver; use crate::npm::NpmFetchResolver; -enum DenoConfigFormat { - Json, - Jsonc, -} +mod cache_deps; +pub(crate) mod deps; +mod outdated; -impl DenoConfigFormat { - fn from_specifier(spec: &ModuleSpecifier) -> Result { - let file_name = spec - .path_segments() - .ok_or_else(|| anyhow!("Empty path in deno config specifier: {spec}"))? - .last() - .unwrap(); - match file_name { - "deno.json" => Ok(Self::Json), - "deno.jsonc" => Ok(Self::Jsonc), - _ => bail!("Unsupported deno config file: {file_name}"), - } - } -} +pub use cache_deps::cache_top_level_deps; +pub use outdated::outdated; -struct DenoConfig { - config: Arc, - format: DenoConfigFormat, - imports: IndexMap, +#[derive(Debug, Copy, Clone, Hash)] +enum ConfigKind { + DenoJson, + PackageJson, } -fn deno_json_imports( - config: &deno_config::deno_json::ConfigFile, -) -> Result, AnyError> { - Ok( - config - .json - .imports - .clone() - .map(|imports| { - serde_json::from_value(imports) - .map_err(|err| anyhow!("Malformed \"imports\" configuration: {err}")) - }) - .transpose()? - .unwrap_or_default(), - ) +struct ConfigUpdater { + kind: ConfigKind, + cst: CstRootNode, + root_object: CstObject, + path: PathBuf, + modified: bool, } -impl DenoConfig { - fn from_options(options: &CliOptions) -> Result, AnyError> { - let start_dir = &options.start_dir; - if let Some(config) = start_dir.maybe_deno_json() { - Ok(Some(Self { - imports: deno_json_imports(config)?, - config: config.clone(), - format: DenoConfigFormat::from_specifier(&config.specifier)?, - })) - } else { - Ok(None) - } - } - fn add(&mut self, selected: SelectedPackage) { - self.imports.insert( - selected.import_name, - format!("{}@{}", selected.package_name, selected.version_req), - ); - } - - fn remove(&mut self, package: &str) -> bool { - self.imports.shift_remove(package).is_some() - } - - fn take_import_fields( - &mut self, - ) -> Vec<(&'static str, IndexMap)> { - vec![("imports", std::mem::take(&mut self.imports))] +impl ConfigUpdater { + fn new( + kind: ConfigKind, + config_file_path: PathBuf, + ) -> Result { + let config_file_contents = std::fs::read_to_string(&config_file_path) + .with_context(|| { + format!("Reading config file '{}'", config_file_path.display()) + })?; + let cst = CstRootNode::parse(&config_file_contents, &Default::default()) + .with_context(|| { + format!("Parsing config file '{}'", config_file_path.display()) + })?; + let root_object = cst.object_value_or_set(); + Ok(Self { + kind, + cst, + root_object, + path: config_file_path, + modified: false, + }) } -} -impl NpmConfig { - fn from_options(options: &CliOptions) -> Result, AnyError> { - let start_dir = &options.start_dir; - if let Some(pkg_json) = start_dir.maybe_pkg_json() { - Ok(Some(Self { - dependencies: pkg_json.dependencies.clone().unwrap_or_default(), - dev_dependencies: pkg_json.dev_dependencies.clone().unwrap_or_default(), - config: pkg_json.clone(), - fmt_options: None, - })) - } else { - Ok(None) - } + fn display_path(&self) -> String { + deno_path_util::url_from_file_path(&self.path) + .map(|u| u.to_string()) + .unwrap_or_else(|_| self.path.display().to_string()) } - fn add(&mut self, selected: SelectedPackage, dev: bool) { - let (name, version) = package_json_dependency_entry(selected); - if dev { - self.dev_dependencies.insert(name, version); - } else { - self.dependencies.insert(name, version); - } + fn obj(&self) -> &CstObject { + &self.root_object } - fn remove(&mut self, package: &str) -> bool { - let in_deps = self.dependencies.shift_remove(package).is_some(); - let in_dev_deps = self.dev_dependencies.shift_remove(package).is_some(); - in_deps || in_dev_deps + fn contents(&self) -> String { + self.cst.to_string() } - fn take_import_fields( + fn get_property_for_mutation( &mut self, - ) -> Vec<(&'static str, IndexMap)> { - vec![ - ("dependencies", std::mem::take(&mut self.dependencies)), - ( - "devDependencies", - std::mem::take(&mut self.dev_dependencies), - ), - ] - } -} + key_path: &KeyPath, + ) -> Option { + let mut current_node = self.root_object.clone(); -struct NpmConfig { - config: Arc, - fmt_options: Option, - dependencies: IndexMap, - dev_dependencies: IndexMap, -} + self.modified = true; -enum DenoOrPackageJson { - Deno(DenoConfig), - Npm(NpmConfig), -} + for (i, part) in key_path.parts.iter().enumerate() { + let s = part.as_str(); + if i < key_path.parts.len().saturating_sub(1) { + let object = current_node.object_value(s)?; + current_node = object; + } else { + // last part + return current_node.get(s); + } + } -impl From for DenoOrPackageJson { - fn from(config: DenoConfig) -> Self { - Self::Deno(config) + None } -} -impl From for DenoOrPackageJson { - fn from(config: NpmConfig) -> Self { - Self::Npm(config) - } -} + fn add(&mut self, selected: SelectedPackage, dev: bool) { + fn insert_index(object: &CstObject, searching_name: &str) -> usize { + object + .properties() + .into_iter() + .take_while(|prop| { + let prop_name = + prop.name().and_then(|name| name.decoded_value().ok()); + match prop_name { + Some(current_name) => { + searching_name.cmp(¤t_name) == std::cmp::Ordering::Greater + } + None => true, + } + }) + .count() + } -/// Wrapper around `jsonc_parser::ast::Object` that can be stored in a `Yoke` -#[derive(yoke::Yokeable)] -struct JsoncObjectView<'a>(jsonc_parser::ast::Object<'a>); + match self.kind { + ConfigKind::DenoJson => { + let imports = self.root_object.object_value_or_set("imports"); + let value = + format!("{}@{}", selected.package_name, selected.version_req); + if let Some(prop) = imports.get(&selected.import_name) { + prop.set_value(json!(value)); + } else { + let index = insert_index(&imports, &selected.import_name); + imports.insert(index, &selected.import_name, json!(value)); + } + } + ConfigKind::PackageJson => { + let deps_prop = self.root_object.get("dependencies"); + let dev_deps_prop = self.root_object.get("devDependencies"); + + let dependencies = if dev { + self + .root_object + .object_value("devDependencies") + .unwrap_or_else(|| { + let index = deps_prop + .as_ref() + .map(|p| p.property_index() + 1) + .unwrap_or_else(|| self.root_object.properties().len()); + self + .root_object + .insert(index, "devDependencies", json!({})) + .object_value_or_set() + }) + } else { + self + .root_object + .object_value("dependencies") + .unwrap_or_else(|| { + let index = dev_deps_prop + .as_ref() + .map(|p| p.property_index()) + .unwrap_or_else(|| self.root_object.properties().len()); + self + .root_object + .insert(index, "dependencies", json!({})) + .object_value_or_set() + }) + }; + let other_dependencies = if dev { + deps_prop.and_then(|p| p.value().and_then(|v| v.as_object())) + } else { + dev_deps_prop.and_then(|p| p.value().and_then(|v| v.as_object())) + }; -struct ConfigUpdater { - config: DenoOrPackageJson, - // the `Yoke` is so we can carry the parsed object (which borrows from - // the source) along with the source itself - ast: Yoke, String>, - path: PathBuf, - modified: bool, -} + let (alias, value) = package_json_dependency_entry(selected); -impl ConfigUpdater { - fn obj(&self) -> &jsonc_parser::ast::Object<'_> { - &self.ast.get().0 - } - fn contents(&self) -> &str { - self.ast.backing_cart() - } - async fn maybe_new( - config: Option>, - ) -> Result, AnyError> { - if let Some(config) = config { - Ok(Some(Self::new(config.into()).await?)) - } else { - Ok(None) - } - } - async fn new(config: DenoOrPackageJson) -> Result { - let specifier = config.specifier(); - if specifier.scheme() != "file" { - bail!("Can't update a remote configuration file"); - } - let config_file_path = specifier.to_file_path().map_err(|_| { - anyhow!("Specifier {specifier:?} is an invalid file path") - })?; - let config_file_contents = { - let contents = tokio::fs::read_to_string(&config_file_path) - .await - .with_context(|| { - format!("Reading config file at: {}", config_file_path.display()) - })?; - if contents.trim().is_empty() { - "{}\n".into() - } else { - contents - } - }; - let ast = Yoke::try_attach_to_cart(config_file_contents, |contents| { - let ast = jsonc_parser::parse_to_ast( - contents, - &Default::default(), - &Default::default(), - ) - .with_context(|| { - format!("Failed to parse config file at {}", specifier) - })?; - let obj = match ast.value { - Some(Value::Object(obj)) => obj, - _ => bail!( - "Failed to update config file at {}, expected an object", - specifier - ), - }; - Ok(JsoncObjectView(obj)) - })?; - Ok(Self { - config, - ast, - path: config_file_path, - modified: false, - }) - } + if let Some(other) = other_dependencies { + if let Some(prop) = other.get(&alias) { + remove_prop_and_maybe_parent_prop(prop); + } + } - fn add(&mut self, selected: SelectedPackage, dev: bool) { - match &mut self.config { - DenoOrPackageJson::Deno(deno) => deno.add(selected), - DenoOrPackageJson::Npm(npm) => npm.add(selected, dev), + if let Some(prop) = dependencies.get(&alias) { + prop.set_value(json!(value)); + } else { + let index = insert_index(&dependencies, &alias); + dependencies.insert(index, &alias, json!(value)); + } + } } + self.modified = true; } fn remove(&mut self, package: &str) -> bool { - let removed = match &mut self.config { - DenoOrPackageJson::Deno(deno) => deno.remove(package), - DenoOrPackageJson::Npm(npm) => npm.remove(package), + let removed = match self.kind { + ConfigKind::DenoJson => { + if let Some(prop) = self + .root_object + .object_value("imports") + .and_then(|i| i.get(package)) + { + remove_prop_and_maybe_parent_prop(prop); + true + } else { + false + } + } + ConfigKind::PackageJson => { + let deps = [ + self + .root_object + .object_value("dependencies") + .and_then(|deps| deps.get(package)), + self + .root_object + .object_value("devDependencies") + .and_then(|deps| deps.get(package)), + ]; + let removed = deps.iter().any(|d| d.is_some()); + for dep in deps.into_iter().flatten() { + remove_prop_and_maybe_parent_prop(dep); + } + removed + } }; if removed { self.modified = true; @@ -273,76 +239,28 @@ impl ConfigUpdater { removed } - async fn commit(mut self) -> Result<(), AnyError> { + fn commit(&self) -> Result<(), AnyError> { if !self.modified { return Ok(()); } - let import_fields = self.config.take_import_fields(); - - let fmt_config_options = self.config.fmt_options(); - - let new_text = update_config_file_content( - self.obj(), - self.contents(), - fmt_config_options, - import_fields.into_iter().map(|(k, v)| { - ( - k, - if v.is_empty() { - None - } else { - Some(generate_imports(v.into_iter().collect())) - }, - ) - }), - self.config.file_name(), - ); - - tokio::fs::write(&self.path, new_text).await?; + let new_text = self.contents(); + std::fs::write(&self.path, new_text).with_context(|| { + format!("failed writing to '{}'", self.path.display()) + })?; Ok(()) } } -impl DenoOrPackageJson { - fn specifier(&self) -> Cow { - match self { - Self::Deno(d, ..) => Cow::Borrowed(&d.config.specifier), - Self::Npm(n, ..) => Cow::Owned(n.config.specifier()), - } - } - - fn fmt_options(&self) -> FmtOptionsConfig { - match self { - DenoOrPackageJson::Deno(deno, ..) => deno - .config - .to_fmt_config() - .ok() - .map(|f| f.options) - .unwrap_or_default(), - DenoOrPackageJson::Npm(config) => { - config.fmt_options.clone().unwrap_or_default() - } - } - } - - fn take_import_fields( - &mut self, - ) -> Vec<(&'static str, IndexMap)> { - match self { - Self::Deno(d) => d.take_import_fields(), - Self::Npm(n) => n.take_import_fields(), - } - } - - fn file_name(&self) -> &'static str { - match self { - DenoOrPackageJson::Deno(config) => match config.format { - DenoConfigFormat::Json => "deno.json", - DenoConfigFormat::Jsonc => "deno.jsonc", - }, - DenoOrPackageJson::Npm(..) => "package.json", - } +fn remove_prop_and_maybe_parent_prop(prop: CstObjectProp) { + let parent = prop.parent().unwrap().as_object().unwrap(); + prop.remove(); + if parent.properties().is_empty() { + let parent_property = parent.parent().unwrap(); + let root_object = parent_property.parent().unwrap().as_object().unwrap(); + // remove the property + parent_property.remove(); + root_object.ensure_multiline(); } } @@ -361,7 +279,14 @@ fn package_json_dependency_entry( selected: SelectedPackage, ) -> (String, String) { if let Some(npm_package) = selected.package_name.strip_prefix("npm:") { - (npm_package.into(), selected.version_req) + if selected.import_name == npm_package { + (npm_package.into(), selected.version_req) + } else { + ( + selected.import_name, + format!("npm:{}@{}", npm_package, selected.version_req), + ) + } } else if let Some(jsr_package) = selected.package_name.strip_prefix("jsr:") { let jsr_package = jsr_package.strip_prefix('@').unwrap_or(jsr_package); let scope_replaced = jsr_package.replace('/', "__"); @@ -391,21 +316,45 @@ impl std::fmt::Display for AddCommandName { fn load_configs( flags: &Arc, -) -> Result<(CliFactory, Option, Option), AnyError> { + has_jsr_specifiers: impl FnOnce() -> bool, +) -> Result<(CliFactory, Option, Option), AnyError> +{ let cli_factory = CliFactory::from_flags(flags.clone()); let options = cli_factory.cli_options()?; - let npm_config = NpmConfig::from_options(options)?; - let (cli_factory, deno_config) = match DenoConfig::from_options(options)? { + let start_dir = &options.start_dir; + let npm_config = match start_dir.maybe_pkg_json() { + Some(pkg_json) => Some(ConfigUpdater::new( + ConfigKind::PackageJson, + pkg_json.path.clone(), + )?), + None => None, + }; + let deno_config = match start_dir.maybe_deno_json() { + Some(deno_json) => Some(ConfigUpdater::new( + ConfigKind::DenoJson, + url_to_file_path(&deno_json.specifier)?, + )?), + None => None, + }; + + let (cli_factory, deno_config) = match deno_config { Some(config) => (cli_factory, Some(config)), - None if npm_config.is_some() => (cli_factory, None), - None => { + None if npm_config.is_some() && !has_jsr_specifiers() => { + (cli_factory, None) + } + _ => { let factory = create_deno_json(flags, options)?; let options = factory.cli_options()?.clone(); + let deno_json = options + .start_dir + .maybe_deno_json() + .expect("Just created deno.json"); ( factory, - Some( - DenoConfig::from_options(&options)?.expect("Just created deno.json"), - ), + Some(ConfigUpdater::new( + ConfigKind::DenoJson, + url_to_file_path(&deno_json.specifier)?, + )?), ) } }; @@ -413,18 +362,26 @@ fn load_configs( Ok((cli_factory, npm_config, deno_config)) } +fn path_distance(a: &Path, b: &Path) -> usize { + let diff = pathdiff::diff_paths(a, b); + let Some(diff) = diff else { + return usize::MAX; + }; + diff.components().count() +} + pub async fn add( flags: Arc, add_flags: AddFlags, cmd_name: AddCommandName, ) -> Result<(), AnyError> { - let (cli_factory, npm_config, deno_config) = load_configs(&flags)?; - let mut npm_config = ConfigUpdater::maybe_new(npm_config).await?; - let mut deno_config = ConfigUpdater::maybe_new(deno_config).await?; + let (cli_factory, mut npm_config, mut deno_config) = + load_configs(&flags, || { + add_flags.packages.iter().any(|s| s.starts_with("jsr:")) + })?; if let Some(deno) = &deno_config { - let specifier = deno.config.specifier(); - if deno.obj().get_string("importMap").is_some() { + if deno.obj().get("importMap").is_some() { bail!( concat!( "`deno {}` is not supported when configuration file contains an \"importMap\" field. ", @@ -432,11 +389,26 @@ pub async fn add( " at {}", ), cmd_name, - specifier + deno.display_path(), ); } } + let start_dir = cli_factory.cli_options()?.start_dir.dir_path(); + + // only prefer to add npm deps to `package.json` if there isn't a closer deno.json. + // example: if deno.json is in the CWD and package.json is in the parent, we should add + // npm deps to deno.json, since it's closer + let prefer_npm_config = match (npm_config.as_ref(), deno_config.as_ref()) { + (Some(npm), Some(deno)) => { + let npm_distance = path_distance(&npm.path, &start_dir); + let deno_distance = path_distance(&deno.path, &start_dir); + npm_distance <= deno_distance + } + (Some(_), None) => true, + (None, _) => false, + }; + let http_client = cli_factory.http_client_provider(); let deps_http_cache = cli_factory.global_http_cache()?; let mut deps_file_fetcher = FileFetcher::new( @@ -447,16 +419,20 @@ pub async fn add( Default::default(), None, ); + + let npmrc = cli_factory.cli_options().unwrap().npmrc(); + deps_file_fetcher.set_download_log_level(log::Level::Trace); let deps_file_fetcher = Arc::new(deps_file_fetcher); let jsr_resolver = Arc::new(JsrFetchResolver::new(deps_file_fetcher.clone())); - let npm_resolver = Arc::new(NpmFetchResolver::new(deps_file_fetcher)); + let npm_resolver = + Arc::new(NpmFetchResolver::new(deps_file_fetcher, npmrc.clone())); let mut selected_packages = Vec::with_capacity(add_flags.packages.len()); let mut package_reqs = Vec::with_capacity(add_flags.packages.len()); for entry_text in add_flags.packages.iter() { - let req = AddPackageReq::parse(entry_text).with_context(|| { + let req = AddRmPackageReq::parse(entry_text).with_context(|| { format!("Failed to parse package required: {}", entry_text) })?; @@ -507,15 +483,32 @@ pub async fn add( match package_and_version { PackageAndVersion::NotFound { package: package_name, - found_npm_package, + help, package_req, - } => { - if found_npm_package { - bail!("{} was not found, but a matching npm package exists. Did you mean `{}`?", crate::colors::red(package_name), crate::colors::yellow(format!("deno {cmd_name} npm:{package_req}"))); - } else { - bail!("{} was not found.", crate::colors::red(package_name)); + } => match help { + Some(NotFoundHelp::NpmPackage) => { + bail!( + "{} was not found, but a matching npm package exists. Did you mean `{}`?", + crate::colors::red(package_name), + crate::colors::yellow(format!("deno {cmd_name} npm:{package_req}")) + ); } - } + Some(NotFoundHelp::JsrPackage) => { + bail!( + "{} was not found, but a matching jsr package exists. Did you mean `{}`?", + crate::colors::red(package_name), + crate::colors::yellow(format!("deno {cmd_name} jsr:{package_req}")) + ) + } + Some(NotFoundHelp::PreReleaseVersion(version)) => { + bail!( + "{} has only pre-release versions available. Try specifying a version: `{}`", + crate::colors::red(&package_name), + crate::colors::yellow(format!("deno {cmd_name} {package_name}@^{version}")) + ) + } + None => bail!("{} was not found.", crate::colors::red(package_name)), + }, PackageAndVersion::Selected(selected) => { selected_packages.push(selected); } @@ -531,7 +524,7 @@ pub async fn add( selected_package.selected_version ); - if selected_package.package_name.starts_with("npm:") { + if selected_package.package_name.starts_with("npm:") && prefer_npm_config { if let Some(npm) = &mut npm_config { npm.add(selected_package, dev); } else { @@ -544,26 +537,14 @@ pub async fn add( } } - let mut commit_futures = vec![]; if let Some(npm) = npm_config { - commit_futures.push(npm.commit()); + npm.commit()?; } if let Some(deno) = deno_config { - commit_futures.push(deno.commit()); + deno.commit()?; } - let commit_futures = - deno_core::futures::future::join_all(commit_futures).await; - for result in commit_futures { - result.context("Failed to update configuration file")?; - } - - // clear the previously cached package.json from memory before reloading it - node_resolver::PackageJsonThreadLocalCache::clear(); - // make a new CliFactory to pick up the updated config file - let cli_factory = CliFactory::from_flags(flags); - // cache deps - cache_deps::cache_top_level_deps(&cli_factory, Some(jsr_resolver)).await?; + npm_install_after_modification(flags, Some(jsr_resolver)).await?; Ok(()) } @@ -575,87 +556,161 @@ struct SelectedPackage { selected_version: String, } +enum NotFoundHelp { + NpmPackage, + JsrPackage, + PreReleaseVersion(Version), +} + enum PackageAndVersion { NotFound { package: String, - found_npm_package: bool, package_req: PackageReq, + help: Option, }, Selected(SelectedPackage), } +fn best_version<'a>( + versions: impl Iterator, +) -> Option<&'a Version> { + let mut maybe_best_version: Option<&Version> = None; + for version in versions { + let is_best_version = maybe_best_version + .as_ref() + .map(|best_version| (*best_version).cmp(version).is_lt()) + .unwrap_or(true); + if is_best_version { + maybe_best_version = Some(version); + } + } + maybe_best_version +} + +trait PackageInfoProvider { + const SPECIFIER_PREFIX: &str; + /// The help to return if a package is found by this provider + const HELP: NotFoundHelp; + async fn req_to_nv(&self, req: &PackageReq) -> Option; + async fn latest_version<'a>(&self, req: &PackageReq) -> Option; +} + +impl PackageInfoProvider for Arc { + const HELP: NotFoundHelp = NotFoundHelp::JsrPackage; + const SPECIFIER_PREFIX: &str = "jsr"; + async fn req_to_nv(&self, req: &PackageReq) -> Option { + (**self).req_to_nv(req).await + } + + async fn latest_version<'a>(&self, req: &PackageReq) -> Option { + let info = self.package_info(&req.name).await?; + best_version( + info + .versions + .iter() + .filter(|(_, version_info)| !version_info.yanked) + .map(|(version, _)| version), + ) + .cloned() + } +} + +impl PackageInfoProvider for Arc { + const HELP: NotFoundHelp = NotFoundHelp::NpmPackage; + const SPECIFIER_PREFIX: &str = "npm"; + async fn req_to_nv(&self, req: &PackageReq) -> Option { + (**self).req_to_nv(req).await + } + + async fn latest_version<'a>(&self, req: &PackageReq) -> Option { + let info = self.package_info(&req.name).await?; + best_version(info.versions.keys()).cloned() + } +} + async fn find_package_and_select_version_for_req( jsr_resolver: Arc, npm_resolver: Arc, - add_package_req: AddPackageReq, + add_package_req: AddRmPackageReq, ) -> Result { - match add_package_req.value { - AddPackageReqValue::Jsr(req) => { - let jsr_prefixed_name = format!("jsr:{}", &req.name); - let Some(nv) = jsr_resolver.req_to_nv(&req).await else { - if npm_resolver.req_to_nv(&req).await.is_some() { + async fn select( + main_resolver: T, + fallback_resolver: S, + add_package_req: AddRmPackageReq, + ) -> Result { + let req = match &add_package_req.value { + AddRmPackageReqValue::Jsr(req) => req, + AddRmPackageReqValue::Npm(req) => req, + }; + let prefixed_name = format!("{}:{}", T::SPECIFIER_PREFIX, req.name); + let help_if_found_in_fallback = S::HELP; + let Some(nv) = main_resolver.req_to_nv(req).await else { + if fallback_resolver.req_to_nv(req).await.is_some() { + // it's in the other registry + return Ok(PackageAndVersion::NotFound { + package: prefixed_name, + help: Some(help_if_found_in_fallback), + package_req: req.clone(), + }); + } + if req.version_req.version_text() == "*" { + if let Some(pre_release_version) = + main_resolver.latest_version(req).await + { return Ok(PackageAndVersion::NotFound { - package: jsr_prefixed_name, - found_npm_package: true, - package_req: req, + package: prefixed_name, + package_req: req.clone(), + help: Some(NotFoundHelp::PreReleaseVersion( + pre_release_version.clone(), + )), }); } + } - return Ok(PackageAndVersion::NotFound { - package: jsr_prefixed_name, - found_npm_package: false, - package_req: req, - }); - }; - let range_symbol = if req.version_req.version_text().starts_with('~') { - '~' - } else { - '^' - }; - Ok(PackageAndVersion::Selected(SelectedPackage { - import_name: add_package_req.alias, - package_name: jsr_prefixed_name, - version_req: format!("{}{}", range_symbol, &nv.version), - selected_version: nv.version.to_string(), - })) + return Ok(PackageAndVersion::NotFound { + package: prefixed_name, + help: None, + package_req: req.clone(), + }); + }; + let range_symbol = if req.version_req.version_text().starts_with('~') { + "~" + } else if req.version_req.version_text() == nv.version.to_string() { + "" + } else { + "^" + }; + Ok(PackageAndVersion::Selected(SelectedPackage { + import_name: add_package_req.alias, + package_name: prefixed_name, + version_req: format!("{}{}", range_symbol, &nv.version), + selected_version: nv.version.to_string(), + })) + } + + match &add_package_req.value { + AddRmPackageReqValue::Jsr(_) => { + select(jsr_resolver, npm_resolver, add_package_req).await } - AddPackageReqValue::Npm(req) => { - let npm_prefixed_name = format!("npm:{}", &req.name); - let Some(nv) = npm_resolver.req_to_nv(&req).await else { - return Ok(PackageAndVersion::NotFound { - package: npm_prefixed_name, - found_npm_package: false, - package_req: req, - }); - }; - let range_symbol = if req.version_req.version_text().starts_with('~') { - '~' - } else { - '^' - }; - Ok(PackageAndVersion::Selected(SelectedPackage { - import_name: add_package_req.alias, - package_name: npm_prefixed_name, - version_req: format!("{}{}", range_symbol, &nv.version), - selected_version: nv.version.to_string(), - })) + AddRmPackageReqValue::Npm(_) => { + select(npm_resolver, jsr_resolver, add_package_req).await } } } #[derive(Debug, PartialEq, Eq)] -enum AddPackageReqValue { +enum AddRmPackageReqValue { Jsr(PackageReq), Npm(PackageReq), } #[derive(Debug, PartialEq, Eq)] -struct AddPackageReq { +pub struct AddRmPackageReq { alias: String, - value: AddPackageReqValue, + value: AddRmPackageReqValue, } -impl AddPackageReq { +impl AddRmPackageReq { pub fn parse(entry_text: &str) -> Result, AnyError> { enum Prefix { Jsr, @@ -710,9 +765,9 @@ impl AddPackageReq { let req_ref = JsrPackageReqReference::from_str(&format!("jsr:{}", entry_text))?; let package_req = req_ref.into_inner().req; - Ok(Ok(AddPackageReq { + Ok(Ok(AddRmPackageReq { alias: maybe_alias.unwrap_or_else(|| package_req.name.to_string()), - value: AddPackageReqValue::Jsr(package_req), + value: AddRmPackageReqValue::Jsr(package_req), })) } Prefix::Npm => { @@ -730,49 +785,48 @@ impl AddPackageReq { deno_semver::RangeSetOrTag::Tag("latest".into()), ); } - Ok(Ok(AddPackageReq { + Ok(Ok(AddRmPackageReq { alias: maybe_alias.unwrap_or_else(|| package_req.name.to_string()), - value: AddPackageReqValue::Npm(package_req), + value: AddRmPackageReqValue::Npm(package_req), })) } } } } -fn generate_imports(mut packages_to_version: Vec<(String, String)>) -> String { - packages_to_version.sort_by(|(k1, _), (k2, _)| k1.cmp(k2)); - let mut contents = vec![]; - let len = packages_to_version.len(); - for (index, (package, version)) in packages_to_version.iter().enumerate() { - // TODO(bartlomieju): fix it, once we start support specifying version on the cli - contents.push(format!("\"{}\": \"{}\"", package, version)); - if index != len - 1 { - contents.push(",".to_string()); - } - } - contents.join("\n") -} - pub async fn remove( flags: Arc, remove_flags: RemoveFlags, ) -> Result<(), AnyError> { - let (_, npm_config, deno_config) = load_configs(&flags)?; + let (_, npm_config, deno_config) = load_configs(&flags, || false)?; - let mut configs = [ - ConfigUpdater::maybe_new(npm_config).await?, - ConfigUpdater::maybe_new(deno_config).await?, - ]; + let mut configs = [npm_config, deno_config]; let mut removed_packages = vec![]; for package in &remove_flags.packages { - let mut removed = false; + let req = AddRmPackageReq::parse(package).with_context(|| { + format!("Failed to parse package required: {}", package) + })?; + let mut parsed_pkg_name = None; for config in configs.iter_mut().flatten() { - removed |= config.remove(package); + match &req { + Ok(rm_pkg) => { + if config.remove(&rm_pkg.alias) && parsed_pkg_name.is_none() { + parsed_pkg_name = Some(rm_pkg.alias.clone()); + } + } + Err(pkg) => { + // An alias or a package name without registry/version + // constraints. Try to remove the package anyway. + if config.remove(&pkg.name) && parsed_pkg_name.is_none() { + parsed_pkg_name = Some(pkg.name.clone()); + } + } + } } - if removed { - removed_packages.push(package.clone()); + if let Some(pkg) = parsed_pkg_name { + removed_packages.push(pkg); } } @@ -783,97 +837,38 @@ pub async fn remove( log::info!("Removed {}", crate::colors::green(package)); } for config in configs.into_iter().flatten() { - config.commit().await?; + config.commit()?; } - // Update deno.lock - node_resolver::PackageJsonThreadLocalCache::clear(); - let cli_factory = CliFactory::from_flags(flags); - cache_deps::cache_top_level_deps(&cli_factory, None).await?; + npm_install_after_modification(flags, None).await?; } Ok(()) } -fn update_config_file_content< - I: IntoIterator)>, ->( - obj: &jsonc_parser::ast::Object, - config_file_contents: &str, - fmt_options: FmtOptionsConfig, - entries: I, - file_name: &str, -) -> String { - let mut text_changes = vec![]; - for (key, value) in entries { - match obj.properties.iter().enumerate().find_map(|(idx, k)| { - if k.name.as_str() == key { - Some((idx, k)) - } else { - None - } - }) { - Some(( - idx, - ObjectProp { - value: Value::Object(lit), - range, - .. - }, - )) => { - if let Some(value) = value { - text_changes.push(TextChange { - range: (lit.range.start + 1)..(lit.range.end - 1), - new_text: value, - }) - } else { - text_changes.push(TextChange { - // remove field entirely, making sure to - // remove the comma if it's not the last field - range: range.start..(if idx == obj.properties.len() - 1 { - range.end - } else { - obj.properties[idx + 1].range.start - }), - new_text: "".to_string(), - }) - } - } +async fn npm_install_after_modification( + flags: Arc, + // explicitly provided to prevent redownloading + jsr_resolver: Option>, +) -> Result { + // clear the previously cached package.json from memory before reloading it + node_resolver::PackageJsonThreadLocalCache::clear(); - // need to add field - None => { - if let Some(value) = value { - let insert_position = obj.range.end - 1; - text_changes.push(TextChange { - range: insert_position..insert_position, - // NOTE(bartlomieju): adding `\n` here to force the formatter to always - // produce a config file that is multiline, like so: - // ``` - // { - // "imports": { - // "": ":@" - // } - // } - new_text: format!("\"{key}\": {{\n {value} }}"), - }) - } - } - // we verified the shape of `imports`/`dependencies` above - Some(_) => unreachable!(), - } + // make a new CliFactory to pick up the updated config file + let cli_factory = CliFactory::from_flags(flags); + // surface any errors in the package.json + let npm_resolver = cli_factory.npm_resolver().await?; + if let Some(npm_resolver) = npm_resolver.as_managed() { + npm_resolver.ensure_no_pkg_json_dep_errors()?; + } + // npm install + cache_deps::cache_top_level_deps(&cli_factory, jsr_resolver).await?; + + if let Some(lockfile) = cli_factory.cli_options()?.maybe_lockfile() { + lockfile.write_if_changed()?; } - let new_text = - deno_ast::apply_text_changes(config_file_contents, text_changes); - - crate::tools::fmt::format_json( - &PathBuf::from(file_name), - &new_text, - &fmt_options, - ) - .ok() - .map(|formatted_text| formatted_text.unwrap_or_else(|| new_text.clone())) - .unwrap_or(new_text) + Ok(cli_factory) } #[cfg(test)] @@ -883,48 +878,52 @@ mod test { #[test] fn test_parse_add_package_req() { assert_eq!( - AddPackageReq::parse("jsr:foo").unwrap().unwrap(), - AddPackageReq { + AddRmPackageReq::parse("jsr:foo").unwrap().unwrap(), + AddRmPackageReq { alias: "foo".to_string(), - value: AddPackageReqValue::Jsr(PackageReq::from_str("foo").unwrap()) + value: AddRmPackageReqValue::Jsr(PackageReq::from_str("foo").unwrap()) } ); assert_eq!( - AddPackageReq::parse("alias@jsr:foo").unwrap().unwrap(), - AddPackageReq { + AddRmPackageReq::parse("alias@jsr:foo").unwrap().unwrap(), + AddRmPackageReq { alias: "alias".to_string(), - value: AddPackageReqValue::Jsr(PackageReq::from_str("foo").unwrap()) + value: AddRmPackageReqValue::Jsr(PackageReq::from_str("foo").unwrap()) } ); assert_eq!( - AddPackageReq::parse("@alias/pkg@npm:foo").unwrap().unwrap(), - AddPackageReq { + AddRmPackageReq::parse("@alias/pkg@npm:foo") + .unwrap() + .unwrap(), + AddRmPackageReq { alias: "@alias/pkg".to_string(), - value: AddPackageReqValue::Npm( + value: AddRmPackageReqValue::Npm( PackageReq::from_str("foo@latest").unwrap() ) } ); assert_eq!( - AddPackageReq::parse("@alias/pkg@jsr:foo").unwrap().unwrap(), - AddPackageReq { + AddRmPackageReq::parse("@alias/pkg@jsr:foo") + .unwrap() + .unwrap(), + AddRmPackageReq { alias: "@alias/pkg".to_string(), - value: AddPackageReqValue::Jsr(PackageReq::from_str("foo").unwrap()) + value: AddRmPackageReqValue::Jsr(PackageReq::from_str("foo").unwrap()) } ); assert_eq!( - AddPackageReq::parse("alias@jsr:foo@^1.5.0") + AddRmPackageReq::parse("alias@jsr:foo@^1.5.0") .unwrap() .unwrap(), - AddPackageReq { + AddRmPackageReq { alias: "alias".to_string(), - value: AddPackageReqValue::Jsr( + value: AddRmPackageReqValue::Jsr( PackageReq::from_str("foo@^1.5.0").unwrap() ) } ); assert_eq!( - AddPackageReq::parse("@scope/pkg@tag") + AddRmPackageReq::parse("@scope/pkg@tag") .unwrap() .unwrap_err() .to_string(), diff --git a/cli/tools/registry/pm/cache_deps.rs b/cli/tools/registry/pm/cache_deps.rs index a598170557dc19..f9d67e4d4f4435 100644 --- a/cli/tools/registry/pm/cache_deps.rs +++ b/cli/tools/registry/pm/cache_deps.rs @@ -8,14 +8,16 @@ use crate::graph_container::ModuleGraphUpdatePermit; use deno_core::error::AnyError; use deno_core::futures::stream::FuturesUnordered; use deno_core::futures::StreamExt; -use deno_semver::package::PackageReq; +use deno_semver::jsr::JsrPackageReqReference; pub async fn cache_top_level_deps( + // todo(dsherret): don't pass the factory into this function. Instead use ctor deps factory: &CliFactory, jsr_resolver: Option>, ) -> Result<(), AnyError> { let npm_resolver = factory.npm_resolver().await?; let cli_options = factory.cli_options()?; + let root_permissions = factory.root_permissions_container()?; if let Some(npm_resolver) = npm_resolver.as_managed() { if !npm_resolver.ensure_top_level_package_json_install().await? { if let Some(lockfile) = cli_options.maybe_lockfile() { @@ -42,7 +44,11 @@ pub async fn cache_top_level_deps( let mut seen_reqs = std::collections::HashSet::new(); - for entry in import_map.imports().entries() { + for entry in import_map.imports().entries().chain( + import_map + .scopes() + .flat_map(|scope| scope.imports.entries()), + ) { let Some(specifier) = entry.value else { continue; }; @@ -50,15 +56,20 @@ pub async fn cache_top_level_deps( match specifier.scheme() { "jsr" => { let specifier_str = specifier.as_str(); - let specifier_str = - specifier_str.strip_prefix("jsr:").unwrap_or(specifier_str); - if let Ok(req) = PackageReq::from_str(specifier_str) { - if !seen_reqs.insert(req.clone()) { + if let Ok(req) = JsrPackageReqReference::from_str(specifier_str) { + if let Some(sub_path) = req.sub_path() { + if sub_path.ends_with('/') { + continue; + } + roots.push(specifier.clone()); + continue; + } + if !seen_reqs.insert(req.req().clone()) { continue; } let jsr_resolver = jsr_resolver.clone(); info_futures.push(async move { - if let Some(nv) = jsr_resolver.req_to_nv(&req).await { + if let Some(nv) = jsr_resolver.req_to_nv(req.req()).await { if let Some(info) = jsr_resolver.package_version_info(&nv).await { return Some((specifier.clone(), info)); @@ -73,6 +84,13 @@ pub async fn cache_top_level_deps( if entry.key.ends_with('/') && specifier.as_str().ends_with('/') { continue; } + if specifier.scheme() == "file" { + if let Ok(path) = specifier.to_file_path() { + if !path.is_file() { + continue; + } + } + } roots.push(specifier.clone()); } } @@ -80,10 +98,6 @@ pub async fn cache_top_level_deps( while let Some(info_future) = info_futures.next().await { if let Some((specifier, info)) = info_future { - if info.export(".").is_some() { - roots.push(specifier.clone()); - continue; - } let exports = info.exports(); for (k, _) in exports { if let Ok(spec) = specifier.join(k) { @@ -106,7 +120,7 @@ pub async fn cache_top_level_deps( &roots, false, deno_config::deno_json::TsTypeLib::DenoWorker, - crate::file_fetcher::FetchPermissionsOption::AllowAll, + root_permissions.clone(), None, ) .await?; diff --git a/cli/tools/registry/pm/deps.rs b/cli/tools/registry/pm/deps.rs new file mode 100644 index 00000000000000..4778d6f3278178 --- /dev/null +++ b/cli/tools/registry/pm/deps.rs @@ -0,0 +1,964 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::borrow::Cow; +use std::collections::HashMap; +use std::sync::atomic::AtomicBool; +use std::sync::Arc; + +use deno_ast::ModuleSpecifier; +use deno_config::deno_json::ConfigFile; +use deno_config::deno_json::ConfigFileRc; +use deno_config::workspace::Workspace; +use deno_config::workspace::WorkspaceDirectory; +use deno_core::anyhow::bail; +use deno_core::error::AnyError; +use deno_core::futures::future::try_join; +use deno_core::futures::stream::FuturesOrdered; +use deno_core::futures::stream::FuturesUnordered; +use deno_core::futures::FutureExt; +use deno_core::futures::StreamExt; +use deno_core::serde_json; +use deno_graph::FillFromLockfileOptions; +use deno_package_json::PackageJsonDepValue; +use deno_package_json::PackageJsonDepValueParseError; +use deno_package_json::PackageJsonRc; +use deno_runtime::deno_permissions::PermissionsContainer; +use deno_semver::jsr::JsrPackageReqReference; +use deno_semver::npm::NpmPackageReqReference; +use deno_semver::package::PackageNv; +use deno_semver::package::PackageReq; +use deno_semver::package::PackageReqReference; +use deno_semver::VersionReq; +use import_map::ImportMap; +use import_map::ImportMapWithDiagnostics; +use import_map::SpecifierMapEntry; +use indexmap::IndexMap; +use tokio::sync::Semaphore; + +use crate::args::CliLockfile; +use crate::graph_container::MainModuleGraphContainer; +use crate::graph_container::ModuleGraphContainer; +use crate::graph_container::ModuleGraphUpdatePermit; +use crate::jsr::JsrFetchResolver; +use crate::module_loader::ModuleLoadPreparer; +use crate::npm::CliNpmResolver; +use crate::npm::NpmFetchResolver; + +use super::ConfigUpdater; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ImportMapKind { + Inline, + Outline, +} + +#[derive(Clone)] +pub enum DepLocation { + DenoJson(ConfigFileRc, KeyPath, ImportMapKind), + PackageJson(PackageJsonRc, KeyPath), +} + +impl DepLocation { + pub fn is_deno_json(&self) -> bool { + matches!(self, DepLocation::DenoJson(..)) + } + + pub fn file_path(&self) -> Cow { + match self { + DepLocation::DenoJson(arc, _, _) => { + Cow::Owned(arc.specifier.to_file_path().unwrap()) + } + DepLocation::PackageJson(arc, _) => Cow::Borrowed(arc.path.as_ref()), + } + } + fn config_kind(&self) -> super::ConfigKind { + match self { + DepLocation::DenoJson(_, _, _) => super::ConfigKind::DenoJson, + DepLocation::PackageJson(_, _) => super::ConfigKind::PackageJson, + } + } +} + +struct DebugAdapter(T); + +impl<'a> std::fmt::Debug for DebugAdapter<&'a ConfigFileRc> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ConfigFile") + .field("specifier", &self.0.specifier) + .finish() + } +} +impl<'a> std::fmt::Debug for DebugAdapter<&'a PackageJsonRc> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PackageJson") + .field("path", &self.0.path) + .finish() + } +} + +impl std::fmt::Debug for DepLocation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + DepLocation::DenoJson(arc, key_path, kind) => { + let mut debug = f.debug_tuple("DenoJson"); + debug + .field(&DebugAdapter(arc)) + .field(key_path) + .field(kind) + .finish() + } + DepLocation::PackageJson(arc, key_path) => { + let mut debug = f.debug_tuple("PackageJson"); + debug.field(&DebugAdapter(arc)).field(key_path).finish() + } + } + } +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum DepKind { + Jsr, + Npm, +} + +impl DepKind { + pub fn scheme(&self) -> &'static str { + match self { + DepKind::Npm => "npm", + DepKind::Jsr => "jsr", + } + } +} + +#[derive(Clone, Debug)] +pub enum KeyPart { + Imports, + Scopes, + Dependencies, + DevDependencies, + String(String), +} + +impl From for KeyPart { + fn from(value: String) -> Self { + KeyPart::String(value) + } +} + +impl From for KeyPart { + fn from(value: PackageJsonDepKind) -> Self { + match value { + PackageJsonDepKind::Normal => Self::Dependencies, + PackageJsonDepKind::Dev => Self::DevDependencies, + } + } +} + +impl KeyPart { + pub fn as_str(&self) -> &str { + match self { + KeyPart::Imports => "imports", + KeyPart::Scopes => "scopes", + KeyPart::Dependencies => "dependencies", + KeyPart::DevDependencies => "devDependencies", + KeyPart::String(s) => s, + } + } +} + +#[derive(Clone, Debug)] +pub struct KeyPath { + pub parts: Vec, +} + +impl KeyPath { + fn from_parts(parts: impl IntoIterator) -> Self { + Self { + parts: parts.into_iter().collect(), + } + } + fn last(&self) -> Option<&KeyPart> { + self.parts.last() + } + fn push(&mut self, part: KeyPart) { + self.parts.push(part) + } +} + +#[derive(Clone, Debug)] +pub struct Dep { + pub req: PackageReq, + pub kind: DepKind, + pub location: DepLocation, + #[allow(dead_code)] + pub id: DepId, + #[allow(dead_code)] + pub alias: Option, +} + +fn import_map_entries( + import_map: &ImportMap, +) -> impl Iterator)> { + import_map + .imports() + .entries() + .map(|entry| { + ( + KeyPath::from_parts([ + KeyPart::Imports, + KeyPart::String(entry.raw_key.into()), + ]), + entry, + ) + }) + .chain(import_map.scopes().flat_map(|scope| { + let path = KeyPath::from_parts([ + KeyPart::Scopes, + scope.raw_key.to_string().into(), + ]); + + scope.imports.entries().map(move |entry| { + let mut full_path = path.clone(); + full_path.push(KeyPart::String(entry.raw_key.to_string())); + (full_path, entry) + }) + })) +} + +fn to_import_map_value_from_imports( + deno_json: &ConfigFile, +) -> serde_json::Value { + let mut value = serde_json::Map::with_capacity(2); + if let Some(imports) = &deno_json.json.imports { + value.insert("imports".to_string(), imports.clone()); + } + if let Some(scopes) = &deno_json.json.scopes { + value.insert("scopes".to_string(), scopes.clone()); + } + serde_json::Value::Object(value) +} + +fn deno_json_import_map( + deno_json: &ConfigFile, +) -> Result, AnyError> { + let (value, kind) = + if deno_json.json.imports.is_some() || deno_json.json.scopes.is_some() { + ( + to_import_map_value_from_imports(deno_json), + ImportMapKind::Inline, + ) + } else { + match deno_json.to_import_map_path()? { + Some(path) => { + let text = std::fs::read_to_string(&path)?; + let value = serde_json::from_str(&text)?; + (value, ImportMapKind::Outline) + } + None => return Ok(None), + } + }; + + import_map::parse_from_value(deno_json.specifier.clone(), value) + .map_err(Into::into) + .map(|import_map| Some((import_map, kind))) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PackageJsonDepKind { + Normal, + Dev, +} + +type PackageJsonDeps = IndexMap< + String, + Result< + (PackageJsonDepKind, PackageJsonDepValue), + PackageJsonDepValueParseError, + >, +>; + +/// Resolve the package.json's dependencies. +// TODO(nathanwhit): Remove once we update deno_package_json with dev deps split out +fn resolve_local_package_json_deps( + package_json: &PackageJsonRc, +) -> PackageJsonDeps { + /// Gets the name and raw version constraint for a registry info or + /// package.json dependency entry taking into account npm package aliases. + fn parse_dep_entry_name_and_raw_version<'a>( + key: &'a str, + value: &'a str, + ) -> (&'a str, &'a str) { + if let Some(package_and_version) = value.strip_prefix("npm:") { + if let Some((name, version)) = package_and_version.rsplit_once('@') { + // if empty, then the name was scoped and there's no version + if name.is_empty() { + (package_and_version, "*") + } else { + (name, version) + } + } else { + (package_and_version, "*") + } + } else { + (key, value) + } + } + + fn parse_entry( + key: &str, + value: &str, + ) -> Result { + if let Some(workspace_key) = value.strip_prefix("workspace:") { + let version_req = VersionReq::parse_from_npm(workspace_key)?; + return Ok(PackageJsonDepValue::Workspace(version_req)); + } + if value.starts_with("file:") + || value.starts_with("git:") + || value.starts_with("http:") + || value.starts_with("https:") + { + return Err(PackageJsonDepValueParseError::Unsupported { + scheme: value.split(':').next().unwrap().to_string(), + }); + } + let (name, version_req) = parse_dep_entry_name_and_raw_version(key, value); + let result = VersionReq::parse_from_npm(version_req); + match result { + Ok(version_req) => Ok(PackageJsonDepValue::Req(PackageReq { + name: name.to_string(), + version_req, + })), + Err(err) => Err(PackageJsonDepValueParseError::VersionReq(err)), + } + } + + fn insert_deps( + deps: Option<&IndexMap>, + result: &mut PackageJsonDeps, + kind: PackageJsonDepKind, + ) { + if let Some(deps) = deps { + for (key, value) in deps { + result.entry(key.to_string()).or_insert_with(|| { + parse_entry(key, value).map(|entry| (kind, entry)) + }); + } + } + } + + let deps = package_json.dependencies.as_ref(); + let dev_deps = package_json.dev_dependencies.as_ref(); + let mut result = IndexMap::new(); + + // favors the deps over dev_deps + insert_deps(deps, &mut result, PackageJsonDepKind::Normal); + insert_deps(dev_deps, &mut result, PackageJsonDepKind::Dev); + + result +} + +fn add_deps_from_deno_json( + deno_json: &Arc, + mut filter: impl DepFilter, + deps: &mut Vec, +) { + let (import_map, import_map_kind) = match deno_json_import_map(deno_json) { + Ok(Some((import_map, import_map_kind))) => (import_map, import_map_kind), + Ok(None) => return, + Err(e) => { + log::warn!("failed to parse imports from {}: {e}", &deno_json.specifier); + return; + } + }; + for (key_path, entry) in import_map_entries(&import_map.import_map) { + let Some(value) = entry.value else { continue }; + let kind = match value.scheme() { + "npm" => DepKind::Npm, + "jsr" => DepKind::Jsr, + _ => continue, + }; + let req = match parse_req_reference(value.as_str(), kind) { + Ok(req) => req.req.clone(), + Err(err) => { + log::warn!("failed to parse package req \"{}\": {err}", value.as_str()); + continue; + } + }; + let alias: &str = key_path.last().unwrap().as_str().trim_end_matches('/'); + let alias = (alias != req.name).then(|| alias.to_string()); + if !filter.should_include(alias.as_deref(), &req, kind) { + continue; + } + let id = DepId(deps.len()); + deps.push(Dep { + location: DepLocation::DenoJson( + deno_json.clone(), + key_path, + import_map_kind, + ), + kind, + req, + id, + alias, + }); + } +} + +fn add_deps_from_package_json( + package_json: &PackageJsonRc, + mut filter: impl DepFilter, + deps: &mut Vec, +) { + let package_json_deps = resolve_local_package_json_deps(package_json); + for (k, v) in package_json_deps { + let (package_dep_kind, v) = match v { + Ok((k, v)) => (k, v), + Err(e) => { + log::warn!("bad package json dep value: {e}"); + continue; + } + }; + match v { + deno_package_json::PackageJsonDepValue::Req(req) => { + let alias = k.as_str(); + let alias = (alias != req.name).then(|| alias.to_string()); + if !filter.should_include(alias.as_deref(), &req, DepKind::Npm) { + continue; + } + let id = DepId(deps.len()); + deps.push(Dep { + id, + kind: DepKind::Npm, + location: DepLocation::PackageJson( + package_json.clone(), + KeyPath::from_parts([package_dep_kind.into(), k.into()]), + ), + req, + alias, + }) + } + deno_package_json::PackageJsonDepValue::Workspace(_) => continue, + } + } +} + +fn deps_from_workspace( + workspace: &Arc, + dep_filter: impl DepFilter, +) -> Result, AnyError> { + let mut deps = Vec::with_capacity(256); + for deno_json in workspace.deno_jsons() { + add_deps_from_deno_json(deno_json, dep_filter, &mut deps); + } + for package_json in workspace.package_jsons() { + add_deps_from_package_json(package_json, dep_filter, &mut deps); + } + + Ok(deps) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct DepId(usize); + +#[derive(Debug, Clone)] +pub enum Change { + Update(DepId, VersionReq), +} + +pub trait DepFilter: Copy { + fn should_include( + &mut self, + alias: Option<&str>, + package_req: &PackageReq, + dep_kind: DepKind, + ) -> bool; +} + +impl DepFilter for T +where + T: FnMut(Option<&str>, &PackageReq, DepKind) -> bool + Copy, +{ + fn should_include<'a>( + &mut self, + alias: Option<&'a str>, + package_req: &'a PackageReq, + dep_kind: DepKind, + ) -> bool { + (*self)(alias, package_req, dep_kind) + } +} + +#[derive(Clone, Debug)] +pub struct PackageLatestVersion { + pub semver_compatible: Option, + pub latest: Option, +} + +pub struct DepManager { + deps: Vec, + resolved_versions: Vec>, + latest_versions: Vec, + + pending_changes: Vec, + + dependencies_resolved: AtomicBool, + module_load_preparer: Arc, + // TODO(nathanwhit): probably shouldn't be pub + pub(crate) jsr_fetch_resolver: Arc, + pub(crate) npm_fetch_resolver: Arc, + npm_resolver: Arc, + permissions_container: PermissionsContainer, + main_module_graph_container: Arc, + lockfile: Option>, +} + +pub struct DepManagerArgs { + pub module_load_preparer: Arc, + pub jsr_fetch_resolver: Arc, + pub npm_fetch_resolver: Arc, + pub npm_resolver: Arc, + pub permissions_container: PermissionsContainer, + pub main_module_graph_container: Arc, + pub lockfile: Option>, +} + +impl DepManager { + pub fn reloaded_after_modification(self, args: DepManagerArgs) -> Self { + let mut new = Self::with_deps_args(self.deps, args); + new.latest_versions = self.latest_versions; + new + } + fn with_deps_args(deps: Vec, args: DepManagerArgs) -> Self { + let DepManagerArgs { + module_load_preparer, + jsr_fetch_resolver, + npm_fetch_resolver, + npm_resolver, + permissions_container, + main_module_graph_container, + lockfile, + } = args; + Self { + deps, + resolved_versions: Vec::new(), + latest_versions: Vec::new(), + jsr_fetch_resolver, + dependencies_resolved: AtomicBool::new(false), + module_load_preparer, + npm_fetch_resolver, + npm_resolver, + permissions_container, + main_module_graph_container, + lockfile, + pending_changes: Vec::new(), + } + } + pub fn from_workspace_dir( + workspace_dir: &Arc, + dep_filter: impl DepFilter, + args: DepManagerArgs, + ) -> Result { + let mut deps = Vec::with_capacity(256); + if let Some(deno_json) = workspace_dir.maybe_deno_json() { + if deno_json.specifier.scheme() != "file" { + bail!("remote deno.json files are not supported"); + } + let path = deno_json.specifier.to_file_path().unwrap(); + if path.parent().unwrap() == workspace_dir.dir_path() { + add_deps_from_deno_json(deno_json, dep_filter, &mut deps); + } + } + if let Some(package_json) = workspace_dir.maybe_pkg_json() { + add_deps_from_package_json(package_json, dep_filter, &mut deps); + } + + Ok(Self::with_deps_args(deps, args)) + } + pub fn from_workspace( + workspace: &Arc, + dep_filter: impl DepFilter, + args: DepManagerArgs, + ) -> Result { + let deps = deps_from_workspace(workspace, dep_filter)?; + Ok(Self::with_deps_args(deps, args)) + } + + async fn run_dependency_resolution(&self) -> Result<(), AnyError> { + if self + .dependencies_resolved + .load(std::sync::atomic::Ordering::Relaxed) + { + return Ok(()); + } + + let mut graph_permit = self + .main_module_graph_container + .acquire_update_permit() + .await; + let graph = graph_permit.graph_mut(); + // populate the information from the lockfile + if let Some(lockfile) = &self.lockfile { + let lockfile = lockfile.lock(); + graph.fill_from_lockfile(FillFromLockfileOptions { + redirects: lockfile + .content + .redirects + .iter() + .map(|(from, to)| (from.as_str(), to.as_str())), + package_specifiers: lockfile + .content + .packages + .specifiers + .iter() + .map(|(dep, id)| (dep, id.as_str())), + }); + } + + let npm_resolver = self.npm_resolver.as_managed().unwrap(); + if self.deps.iter().all(|dep| match dep.kind { + DepKind::Npm => { + npm_resolver.resolve_pkg_id_from_pkg_req(&dep.req).is_ok() + } + DepKind::Jsr => graph.packages.mappings().contains_key(&dep.req), + }) { + self + .dependencies_resolved + .store(true, std::sync::atomic::Ordering::Relaxed); + graph_permit.commit(); + return Ok(()); + } + + npm_resolver.ensure_top_level_package_json_install().await?; + let mut roots = Vec::new(); + let mut info_futures = FuturesUnordered::new(); + for dep in &self.deps { + if dep.location.is_deno_json() { + match dep.kind { + DepKind::Npm => roots.push( + ModuleSpecifier::parse(&format!("npm:/{}/", dep.req)).unwrap(), + ), + DepKind::Jsr => info_futures.push(async { + if let Some(nv) = self.jsr_fetch_resolver.req_to_nv(&dep.req).await + { + if let Some(info) = + self.jsr_fetch_resolver.package_version_info(&nv).await + { + let specifier = + ModuleSpecifier::parse(&format!("jsr:/{}/", dep.req)) + .unwrap(); + return Some((specifier, info)); + } + } + None + }), + } + } + } + + while let Some(info_future) = info_futures.next().await { + if let Some((specifier, info)) = info_future { + let exports = info.exports(); + for (k, _) in exports { + if let Ok(spec) = specifier.join(k) { + roots.push(spec); + } + } + } + } + + self + .module_load_preparer + .prepare_module_load( + graph, + &roots, + false, + deno_config::deno_json::TsTypeLib::DenoWindow, + self.permissions_container.clone(), + None, + ) + .await?; + + graph_permit.commit(); + + Ok(()) + } + + pub fn resolved_version(&self, id: DepId) -> Option<&PackageNv> { + self.resolved_versions[id.0].as_ref() + } + + pub async fn resolve_current_versions(&mut self) -> Result<(), AnyError> { + self.run_dependency_resolution().await?; + + let graph = self.main_module_graph_container.graph(); + + let mut resolved = Vec::with_capacity(self.deps.len()); + let snapshot = self.npm_resolver.as_managed().unwrap().snapshot(); + let resolved_npm = snapshot.package_reqs(); + let resolved_jsr = graph.packages.mappings(); + for dep in &self.deps { + match dep.kind { + DepKind::Npm => { + let resolved_version = resolved_npm.get(&dep.req).cloned(); + resolved.push(resolved_version); + } + DepKind::Jsr => { + let resolved_version = resolved_jsr.get(&dep.req).cloned(); + resolved.push(resolved_version) + } + } + } + + self.resolved_versions = resolved; + + Ok(()) + } + + async fn load_latest_versions( + &self, + ) -> Result, AnyError> { + if self.latest_versions.len() == self.deps.len() { + return Ok(self.latest_versions.clone()); + } + let latest_tag_req = deno_semver::VersionReq::from_raw_text_and_inner( + "latest".into(), + deno_semver::RangeSetOrTag::Tag("latest".into()), + ); + let mut latest_versions = Vec::with_capacity(self.deps.len()); + + let npm_sema = Semaphore::new(32); + let jsr_sema = Semaphore::new(32); + let mut futs = FuturesOrdered::new(); + + for dep in &self.deps { + match dep.kind { + DepKind::Npm => futs.push_back( + async { + let semver_req = &dep.req; + let latest_req = PackageReq { + name: dep.req.name.clone(), + version_req: latest_tag_req.clone(), + }; + let _permit = npm_sema.acquire().await; + let semver_compatible = + self.npm_fetch_resolver.req_to_nv(semver_req).await; + let latest = self.npm_fetch_resolver.req_to_nv(&latest_req).await; + PackageLatestVersion { + latest, + semver_compatible, + } + } + .boxed_local(), + ), + DepKind::Jsr => futs.push_back( + async { + let semver_req = &dep.req; + let latest_req = PackageReq { + name: dep.req.name.clone(), + version_req: deno_semver::WILDCARD_VERSION_REQ.clone(), + }; + let _permit = jsr_sema.acquire().await; + let semver_compatible = + self.jsr_fetch_resolver.req_to_nv(semver_req).await; + let latest = self.jsr_fetch_resolver.req_to_nv(&latest_req).await; + PackageLatestVersion { + latest, + semver_compatible, + } + } + .boxed_local(), + ), + } + } + while let Some(nv) = futs.next().await { + latest_versions.push(nv); + } + + Ok(latest_versions) + } + + pub async fn resolve_versions(&mut self) -> Result<(), AnyError> { + let (_, latest_versions) = try_join( + self.run_dependency_resolution(), + self.load_latest_versions(), + ) + .await?; + + self.latest_versions = latest_versions; + + self.resolve_current_versions().await?; + + Ok(()) + } + + pub fn deps_with_resolved_latest_versions( + &self, + ) -> impl IntoIterator, PackageLatestVersion)> + '_ + { + self + .resolved_versions + .iter() + .zip(self.latest_versions.iter()) + .enumerate() + .map(|(i, (resolved, latest))| { + (DepId(i), resolved.clone(), latest.clone()) + }) + } + + pub fn get_dep(&self, id: DepId) -> &Dep { + &self.deps[id.0] + } + + pub fn update_dep(&mut self, dep_id: DepId, new_version_req: VersionReq) { + self + .pending_changes + .push(Change::Update(dep_id, new_version_req)); + } + + pub fn commit_changes(&mut self) -> Result<(), AnyError> { + let changes = std::mem::take(&mut self.pending_changes); + let mut config_updaters = HashMap::new(); + for change in changes { + match change { + Change::Update(dep_id, version_req) => { + // TODO: move most of this to ConfigUpdater + let dep = &mut self.deps[dep_id.0]; + dep.req.version_req = version_req.clone(); + match &dep.location { + DepLocation::DenoJson(arc, key_path, import_map_kind) => { + if matches!(import_map_kind, ImportMapKind::Outline) { + // not supported + continue; + } + let updater = + get_or_create_updater(&mut config_updaters, &dep.location)?; + + let Some(property) = updater.get_property_for_mutation(key_path) + else { + log::warn!( + "failed to find property at path {key_path:?} for file {}", + arc.specifier + ); + continue; + }; + let Some(string_value) = cst_string_literal(&property) else { + continue; + }; + let mut req_reference = match dep.kind { + DepKind::Npm => NpmPackageReqReference::from_str(&string_value) + .unwrap() + .into_inner(), + DepKind::Jsr => JsrPackageReqReference::from_str(&string_value) + .unwrap() + .into_inner(), + }; + req_reference.req.version_req = version_req; + let mut new_value = + format!("{}:{}", dep.kind.scheme(), req_reference); + if string_value.ends_with('/') && !new_value.ends_with('/') { + // the display impl for PackageReqReference maps `/` to the root + // subpath, but for the import map the trailing `/` is significant + new_value.push('/'); + } + if string_value + .trim_start_matches(format!("{}:", dep.kind.scheme()).as_str()) + .starts_with('/') + { + // this is gross + new_value = new_value.replace(':', ":/"); + } + property + .set_value(jsonc_parser::cst::CstInputValue::String(new_value)); + } + DepLocation::PackageJson(arc, key_path) => { + let updater = + get_or_create_updater(&mut config_updaters, &dep.location)?; + let Some(property) = updater.get_property_for_mutation(key_path) + else { + log::warn!( + "failed to find property at path {key_path:?} for file {}", + arc.path.display() + ); + continue; + }; + let Some(string_value) = cst_string_literal(&property) else { + continue; + }; + let new_value = if string_value.starts_with("npm:") { + // aliased + let rest = string_value.trim_start_matches("npm:"); + let mut parts = rest.split('@'); + let first = parts.next().unwrap(); + if first.is_empty() { + let scope_and_name = parts.next().unwrap(); + format!("npm:@{scope_and_name}@{version_req}") + } else { + format!("npm:{first}@{version_req}") + } + } else if string_value.contains(":") { + bail!("Unexpected package json dependency string: \"{string_value}\" in {}", arc.path.display()); + } else { + version_req.to_string() + }; + property + .set_value(jsonc_parser::cst::CstInputValue::String(new_value)); + } + } + } + } + } + + for (_, updater) in config_updaters { + updater.commit()?; + } + + Ok(()) + } +} + +fn get_or_create_updater<'a>( + config_updaters: &'a mut HashMap, + location: &DepLocation, +) -> Result<&'a mut ConfigUpdater, AnyError> { + match config_updaters.entry(location.file_path().into_owned()) { + std::collections::hash_map::Entry::Occupied(occupied_entry) => { + Ok(occupied_entry.into_mut()) + } + std::collections::hash_map::Entry::Vacant(vacant_entry) => { + let updater = ConfigUpdater::new( + location.config_kind(), + location.file_path().into_owned(), + )?; + Ok(vacant_entry.insert(updater)) + } + } +} + +fn cst_string_literal( + property: &jsonc_parser::cst::CstObjectProp, +) -> Option { + // TODO(nathanwhit): ensure this unwrap is safe + let value = property.value().unwrap(); + let Some(string) = value.as_string_lit() else { + log::warn!("malformed entry"); + return None; + }; + let Ok(string_value) = string.decoded_value() else { + log::warn!("malformed string: {string:?}"); + return None; + }; + Some(string_value) +} + +fn parse_req_reference( + input: &str, + kind: DepKind, +) -> Result< + PackageReqReference, + deno_semver::package::PackageReqReferenceParseError, +> { + Ok(match kind { + DepKind::Npm => NpmPackageReqReference::from_str(input)?.into_inner(), + DepKind::Jsr => JsrPackageReqReference::from_str(input)?.into_inner(), + }) +} diff --git a/cli/tools/registry/pm/outdated.rs b/cli/tools/registry/pm/outdated.rs new file mode 100644 index 00000000000000..2a29014267d67e --- /dev/null +++ b/cli/tools/registry/pm/outdated.rs @@ -0,0 +1,661 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::collections::HashSet; +use std::sync::Arc; + +use deno_core::error::AnyError; +use deno_semver::package::PackageNv; +use deno_semver::package::PackageReq; +use deno_semver::VersionReq; +use deno_terminal::colors; + +use crate::args::CacheSetting; +use crate::args::CliOptions; +use crate::args::Flags; +use crate::args::OutdatedFlags; +use crate::factory::CliFactory; +use crate::file_fetcher::FileFetcher; +use crate::jsr::JsrFetchResolver; +use crate::npm::NpmFetchResolver; +use crate::tools::registry::pm::deps::DepKind; + +use super::deps::Dep; +use super::deps::DepManager; +use super::deps::DepManagerArgs; +use super::deps::PackageLatestVersion; + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] +struct OutdatedPackage { + kind: DepKind, + latest: String, + semver_compatible: String, + current: String, + name: String, +} + +#[allow(clippy::print_stdout)] +fn print_outdated_table(packages: &[OutdatedPackage]) { + const HEADINGS: &[&str] = &["Package", "Current", "Update", "Latest"]; + + let mut longest_package = 0; + let mut longest_current = 0; + let mut longest_update = 0; + let mut longest_latest = 0; + + for package in packages { + let name_len = package.kind.scheme().len() + 1 + package.name.len(); + longest_package = longest_package.max(name_len); + longest_current = longest_current.max(package.current.len()); + longest_update = longest_update.max(package.semver_compatible.len()); + longest_latest = longest_latest.max(package.latest.len()); + } + + let package_column_width = longest_package.max(HEADINGS[0].len()) + 2; + let current_column_width = longest_current.max(HEADINGS[1].len()) + 2; + let update_column_width = longest_update.max(HEADINGS[2].len()) + 2; + let latest_column_width = longest_latest.max(HEADINGS[3].len()) + 2; + + let package_fill = "─".repeat(package_column_width); + let current_fill = "─".repeat(current_column_width); + let update_fill = "─".repeat(update_column_width); + let latest_fill = "─".repeat(latest_column_width); + + println!("┌{package_fill}┬{current_fill}┬{update_fill}┬{latest_fill}┐"); + println!( + "│ {}{} │ {}{} │ {}{} │ {}{} │", + colors::intense_blue(HEADINGS[0]), + " ".repeat(package_column_width - 2 - HEADINGS[0].len()), + colors::intense_blue(HEADINGS[1]), + " ".repeat(current_column_width - 2 - HEADINGS[1].len()), + colors::intense_blue(HEADINGS[2]), + " ".repeat(update_column_width - 2 - HEADINGS[2].len()), + colors::intense_blue(HEADINGS[3]), + " ".repeat(latest_column_width - 2 - HEADINGS[3].len()) + ); + for package in packages { + println!("├{package_fill}┼{current_fill}┼{update_fill}┼{latest_fill}┤",); + + print!( + "│ {: Result<(), AnyError> { + let mut outdated = Vec::new(); + let mut seen = std::collections::BTreeSet::new(); + for (dep_id, resolved, latest_versions) in + deps.deps_with_resolved_latest_versions() + { + let dep = deps.get_dep(dep_id); + + let Some(resolved) = resolved else { continue }; + + let latest = { + let preferred = if compatible { + &latest_versions.semver_compatible + } else { + &latest_versions.latest + }; + if let Some(v) = preferred { + v + } else { + continue; + } + }; + + if latest > &resolved + && seen.insert((dep.kind, dep.req.name.clone(), resolved.version.clone())) + { + outdated.push(OutdatedPackage { + kind: dep.kind, + name: dep.req.name.clone(), + current: resolved.version.to_string(), + latest: latest_versions + .latest + .map(|l| l.version.to_string()) + .unwrap_or_default(), + semver_compatible: latest_versions + .semver_compatible + .map(|l| l.version.to_string()) + .unwrap_or_default(), + }) + } + } + + if !outdated.is_empty() { + outdated.sort(); + print_outdated_table(&outdated); + } + + Ok(()) +} + +pub async fn outdated( + flags: Arc, + update_flags: OutdatedFlags, +) -> Result<(), AnyError> { + let factory = CliFactory::from_flags(flags.clone()); + let cli_options = factory.cli_options()?; + let workspace = cli_options.workspace(); + let http_client = factory.http_client_provider(); + let deps_http_cache = factory.global_http_cache()?; + let mut file_fetcher = FileFetcher::new( + deps_http_cache.clone(), + CacheSetting::RespectHeaders, + true, + http_client.clone(), + Default::default(), + None, + ); + file_fetcher.set_download_log_level(log::Level::Trace); + let file_fetcher = Arc::new(file_fetcher); + let npm_fetch_resolver = Arc::new(NpmFetchResolver::new( + file_fetcher.clone(), + cli_options.npmrc().clone(), + )); + let jsr_fetch_resolver = + Arc::new(JsrFetchResolver::new(file_fetcher.clone())); + + let args = dep_manager_args( + &factory, + cli_options, + npm_fetch_resolver.clone(), + jsr_fetch_resolver.clone(), + ) + .await?; + + let filter_set = filter::FilterSet::from_filter_strings( + update_flags.filters.iter().map(|s| s.as_str()), + )?; + + let filter_fn = |alias: Option<&str>, req: &PackageReq, _: DepKind| { + if filter_set.is_empty() { + return true; + } + let name = alias.unwrap_or(&req.name); + filter_set.matches(name) + }; + let mut deps = if update_flags.recursive { + super::deps::DepManager::from_workspace(workspace, filter_fn, args)? + } else { + super::deps::DepManager::from_workspace_dir( + &cli_options.start_dir, + filter_fn, + args, + )? + }; + + deps.resolve_versions().await?; + + match update_flags.kind { + crate::args::OutdatedKind::Update { latest } => { + update(deps, latest, &filter_set, flags).await?; + } + crate::args::OutdatedKind::PrintOutdated { compatible } => { + print_outdated(&mut deps, compatible)?; + } + } + + Ok(()) +} + +fn choose_new_version_req( + dep: &Dep, + resolved: Option<&PackageNv>, + latest_versions: &PackageLatestVersion, + update_to_latest: bool, + filter_set: &filter::FilterSet, +) -> Option { + let explicit_version_req = filter_set + .matching_filter(dep.alias.as_deref().unwrap_or(&dep.req.name)) + .version_spec() + .cloned(); + + if let Some(version_req) = explicit_version_req { + if let Some(resolved) = resolved { + // todo(nathanwhit): handle tag + if version_req.tag().is_none() && version_req.matches(&resolved.version) { + return None; + } + } + Some(version_req) + } else { + let preferred = if update_to_latest { + latest_versions.latest.as_ref()? + } else { + latest_versions.semver_compatible.as_ref()? + }; + if preferred.version <= resolved?.version { + return None; + } + Some( + VersionReq::parse_from_specifier( + format!("^{}", preferred.version).as_str(), + ) + .unwrap(), + ) + } +} + +async fn update( + mut deps: DepManager, + update_to_latest: bool, + filter_set: &filter::FilterSet, + flags: Arc, +) -> Result<(), AnyError> { + let mut updated = Vec::new(); + + for (dep_id, resolved, latest_versions) in deps + .deps_with_resolved_latest_versions() + .into_iter() + .collect::>() + { + let dep = deps.get_dep(dep_id); + let new_version_req = choose_new_version_req( + dep, + resolved.as_ref(), + &latest_versions, + update_to_latest, + filter_set, + ); + let Some(new_version_req) = new_version_req else { + continue; + }; + + updated.push(( + dep_id, + format!("{}:{}", dep.kind.scheme(), dep.req.name), + deps.resolved_version(dep.id).cloned(), + new_version_req.clone(), + )); + + deps.update_dep(dep_id, new_version_req); + } + + deps.commit_changes()?; + + if !updated.is_empty() { + let factory = super::npm_install_after_modification( + flags.clone(), + Some(deps.jsr_fetch_resolver.clone()), + ) + .await?; + + let mut updated_to_versions = HashSet::new(); + let cli_options = factory.cli_options()?; + let args = dep_manager_args( + &factory, + cli_options, + deps.npm_fetch_resolver.clone(), + deps.jsr_fetch_resolver.clone(), + ) + .await?; + + let mut deps = deps.reloaded_after_modification(args); + deps.resolve_current_versions().await?; + for (dep_id, package_name, maybe_current_version, new_version_req) in + updated + { + if let Some(nv) = deps.resolved_version(dep_id) { + updated_to_versions.insert(( + package_name, + maybe_current_version, + nv.version.clone(), + )); + } else { + log::warn!( + "Failed to resolve version for new version requirement: {} -> {}", + package_name, + new_version_req + ); + } + } + + log::info!( + "Updated {} dependenc{}:", + updated_to_versions.len(), + if updated_to_versions.len() == 1 { + "y" + } else { + "ies" + } + ); + let mut updated_to_versions = + updated_to_versions.into_iter().collect::>(); + updated_to_versions.sort_by(|(k, _, _), (k2, _, _)| k.cmp(k2)); + let max_name = updated_to_versions + .iter() + .map(|(name, _, _)| name.len()) + .max() + .unwrap_or(0); + let max_old = updated_to_versions + .iter() + .map(|(_, maybe_current, _)| { + maybe_current + .as_ref() + .map(|v| v.version.to_string().len()) + .unwrap_or(0) + }) + .max() + .unwrap_or(0); + let max_new = updated_to_versions + .iter() + .map(|(_, _, new_version)| new_version.to_string().len()) + .max() + .unwrap_or(0); + + for (package_name, maybe_current_version, new_version) in + updated_to_versions + { + let current_version = if let Some(current_version) = maybe_current_version + { + current_version.version.to_string() + } else { + "".to_string() + }; + + log::info!( + " - {}{} {}{} -> {}{}", + format!( + "{}{}", + colors::gray(package_name[0..4].to_string()), + package_name[4..].to_string() + ), + " ".repeat(max_name - package_name.len()), + " ".repeat(max_old - current_version.len()), + colors::gray(¤t_version), + " ".repeat(max_new - new_version.to_string().len()), + colors::green(&new_version), + ); + } + } else { + log::info!( + "All {}dependencies are up to date.", + if filter_set.is_empty() { + "" + } else { + "matching " + } + ); + } + + Ok(()) +} + +async fn dep_manager_args( + factory: &CliFactory, + cli_options: &CliOptions, + npm_fetch_resolver: Arc, + jsr_fetch_resolver: Arc, +) -> Result { + Ok(DepManagerArgs { + module_load_preparer: factory.module_load_preparer().await?.clone(), + jsr_fetch_resolver, + npm_fetch_resolver, + npm_resolver: factory.npm_resolver().await?.clone(), + permissions_container: factory.root_permissions_container()?.clone(), + main_module_graph_container: factory + .main_module_graph_container() + .await? + .clone(), + lockfile: cli_options.maybe_lockfile().cloned(), + }) +} + +mod filter { + use deno_core::anyhow::anyhow; + use deno_core::anyhow::Context; + use deno_core::error::AnyError; + use deno_semver::VersionReq; + + enum FilterKind { + Exclude, + Include, + } + pub struct Filter { + kind: FilterKind, + regex: regex::Regex, + version_spec: Option, + } + + fn pattern_to_regex(pattern: &str) -> Result { + let escaped = regex::escape(pattern); + let unescaped_star = escaped.replace(r"\*", ".*"); + Ok(regex::Regex::new(&format!("^{}$", unescaped_star))?) + } + + impl Filter { + pub fn version_spec(&self) -> Option<&VersionReq> { + self.version_spec.as_ref() + } + pub fn from_str(input: &str) -> Result { + let (kind, first_idx) = if input.starts_with('!') { + (FilterKind::Exclude, 1) + } else { + (FilterKind::Include, 0) + }; + let s = &input[first_idx..]; + let (pattern, version_spec) = + if let Some(scope_name) = s.strip_prefix('@') { + if let Some(idx) = scope_name.find('@') { + let (pattern, version_spec) = s.split_at(idx + 1); + ( + pattern, + Some( + VersionReq::parse_from_specifier( + version_spec.trim_start_matches('@'), + ) + .with_context(|| format!("Invalid filter \"{input}\""))?, + ), + ) + } else { + (s, None) + } + } else { + let mut parts = s.split('@'); + let Some(pattern) = parts.next() else { + return Err(anyhow!("Invalid filter \"{input}\"")); + }; + ( + pattern, + parts + .next() + .map(VersionReq::parse_from_specifier) + .transpose() + .with_context(|| format!("Invalid filter \"{input}\""))?, + ) + }; + + Ok(Filter { + kind, + regex: pattern_to_regex(pattern) + .with_context(|| format!("Invalid filter \"{input}\""))?, + version_spec, + }) + } + + pub fn matches(&self, name: &str) -> bool { + self.regex.is_match(name) + } + } + + pub struct FilterSet { + filters: Vec, + has_exclude: bool, + has_include: bool, + } + impl FilterSet { + pub fn from_filter_strings<'a>( + filter_strings: impl IntoIterator, + ) -> Result { + let filters = filter_strings + .into_iter() + .map(Filter::from_str) + .collect::, _>>()?; + let has_exclude = filters + .iter() + .any(|f| matches!(f.kind, FilterKind::Exclude)); + let has_include = filters + .iter() + .any(|f| matches!(f.kind, FilterKind::Include)); + Ok(FilterSet { + filters, + has_exclude, + has_include, + }) + } + + pub fn is_empty(&self) -> bool { + self.filters.is_empty() + } + + pub fn matches(&self, name: &str) -> bool { + self.matching_filter(name).is_included() + } + + pub fn matching_filter(&self, name: &str) -> MatchResult<'_> { + if self.filters.is_empty() { + return MatchResult::Included; + } + let mut matched = None; + for filter in &self.filters { + match filter.kind { + FilterKind::Include => { + if matched.is_none() && filter.matches(name) { + matched = Some(filter); + } + } + FilterKind::Exclude => { + if filter.matches(name) { + return MatchResult::Excluded; + } + } + } + } + if let Some(filter) = matched { + MatchResult::Matches(filter) + } else if self.has_exclude && !self.has_include { + MatchResult::Included + } else { + MatchResult::Excluded + } + } + } + + pub enum MatchResult<'a> { + Matches(&'a Filter), + Included, + Excluded, + } + + impl MatchResult<'_> { + pub fn version_spec(&self) -> Option<&VersionReq> { + match self { + MatchResult::Matches(filter) => filter.version_spec(), + _ => None, + } + } + pub fn is_included(&self) -> bool { + matches!(self, MatchResult::Included | MatchResult::Matches(_)) + } + } + + #[cfg(test)] + mod test { + fn matches_filters<'a, 'b>( + filters: impl IntoIterator, + name: &str, + ) -> bool { + let filters = super::FilterSet::from_filter_strings(filters).unwrap(); + filters.matches(name) + } + + fn version_spec(s: &str) -> deno_semver::VersionReq { + deno_semver::VersionReq::parse_from_specifier(s).unwrap() + } + + #[test] + fn basic_glob() { + assert!(matches_filters(["foo*"], "foo")); + assert!(matches_filters(["foo*"], "foobar")); + assert!(!matches_filters(["foo*"], "barfoo")); + + assert!(matches_filters(["*foo"], "foo")); + assert!(matches_filters(["*foo"], "barfoo")); + assert!(!matches_filters(["*foo"], "foobar")); + + assert!(matches_filters(["@scope/foo*"], "@scope/foobar")); + } + + #[test] + fn basic_glob_with_version() { + assert!(matches_filters(["foo*@1"], "foo",)); + assert!(matches_filters(["foo*@1"], "foobar",)); + assert!(matches_filters(["foo*@1"], "foo-bar",)); + assert!(!matches_filters(["foo*@1"], "barfoo",)); + assert!(matches_filters(["@scope/*@1"], "@scope/foo")); + } + + #[test] + fn glob_exclude() { + assert!(!matches_filters(["!foo*"], "foo")); + assert!(!matches_filters(["!foo*"], "foobar")); + assert!(matches_filters(["!foo*"], "barfoo")); + + assert!(!matches_filters(["!*foo"], "foo")); + assert!(!matches_filters(["!*foo"], "barfoo")); + assert!(matches_filters(["!*foo"], "foobar")); + + assert!(!matches_filters(["!@scope/foo*"], "@scope/foobar")); + } + + #[test] + fn multiple_globs() { + assert!(matches_filters(["foo*", "bar*"], "foo")); + assert!(matches_filters(["foo*", "bar*"], "bar")); + assert!(!matches_filters(["foo*", "bar*"], "baz")); + + assert!(matches_filters(["foo*", "!bar*"], "foo")); + assert!(!matches_filters(["foo*", "!bar*"], "bar")); + assert!(matches_filters(["foo*", "!bar*"], "foobar")); + assert!(!matches_filters(["foo*", "!*bar"], "foobar")); + assert!(!matches_filters(["foo*", "!*bar"], "baz")); + + let filters = + super::FilterSet::from_filter_strings(["foo*@1", "bar*@2"]).unwrap(); + + assert_eq!( + filters.matching_filter("foo").version_spec().cloned(), + Some(version_spec("1")) + ); + + assert_eq!( + filters.matching_filter("bar").version_spec().cloned(), + Some(version_spec("2")) + ); + } + } +} diff --git a/cli/tools/registry/tar.rs b/cli/tools/registry/tar.rs index aca125e00b7708..6d1801ce6976fc 100644 --- a/cli/tools/registry/tar.rs +++ b/cli/tools/registry/tar.rs @@ -120,7 +120,7 @@ fn resolve_content_maybe_unfurling( | MediaType::Unknown | MediaType::Json | MediaType::Wasm - | MediaType::TsBuildInfo => { + | MediaType::Css => { // not unfurlable data return Ok(data); } diff --git a/cli/tools/registry/unfurl.rs b/cli/tools/registry/unfurl.rs index 0f5b9fdd329aa8..90343ac6568d96 100644 --- a/cli/tools/registry/unfurl.rs +++ b/cli/tools/registry/unfurl.rs @@ -12,9 +12,10 @@ use deno_graph::DynamicTemplatePart; use deno_graph::ParserModuleAnalyzer; use deno_graph::TypeScriptReference; use deno_package_json::PackageJsonDepValue; +use deno_resolver::sloppy_imports::SloppyImportsResolutionKind; use deno_runtime::deno_node::is_builtin_node_module; -use crate::resolver::SloppyImportsResolver; +use crate::resolver::CliSloppyImportsResolver; #[derive(Debug, Clone)] pub enum SpecifierUnfurlerDiagnostic { @@ -42,14 +43,14 @@ impl SpecifierUnfurlerDiagnostic { } pub struct SpecifierUnfurler { - sloppy_imports_resolver: Option, + sloppy_imports_resolver: Option, workspace_resolver: WorkspaceResolver, bare_node_builtins: bool, } impl SpecifierUnfurler { pub fn new( - sloppy_imports_resolver: Option, + sloppy_imports_resolver: Option, workspace_resolver: WorkspaceResolver, bare_node_builtins: bool, ) -> Self { @@ -179,7 +180,7 @@ impl SpecifierUnfurler { let resolved = if let Some(sloppy_imports_resolver) = &self.sloppy_imports_resolver { sloppy_imports_resolver - .resolve(&resolved, deno_graph::source::ResolutionMode::Execution) + .resolve(&resolved, SloppyImportsResolutionKind::Execution) .map(|res| res.into_specifier()) .unwrap_or(resolved) } else { @@ -318,8 +319,8 @@ impl SpecifierUnfurler { } for ts_ref in &module_info.ts_references { let specifier_with_range = match ts_ref { - TypeScriptReference::Path(range) => range, - TypeScriptReference::Types(range) => range, + TypeScriptReference::Path(s) => s, + TypeScriptReference::Types { specifier, .. } => specifier, }; analyze_specifier( &specifier_with_range.text, @@ -327,10 +328,10 @@ impl SpecifierUnfurler { &mut text_changes, ); } - for specifier_with_range in &module_info.jsdoc_imports { + for jsdoc in &module_info.jsdoc_imports { analyze_specifier( - &specifier_with_range.text, - &specifier_with_range.range, + &jsdoc.specifier.text, + &jsdoc.specifier.range, &mut text_changes, ); } @@ -388,6 +389,8 @@ fn to_range( mod tests { use std::sync::Arc; + use crate::resolver::SloppyImportsCachedFs; + use super::*; use deno_ast::MediaType; use deno_ast::ModuleSpecifier; @@ -455,7 +458,9 @@ mod tests { ); let fs = Arc::new(RealFs); let unfurler = SpecifierUnfurler::new( - Some(SloppyImportsResolver::new(fs)), + Some(CliSloppyImportsResolver::new(SloppyImportsCachedFs::new( + fs, + ))), workspace_resolver, true, ); diff --git a/cli/tools/repl/mod.rs b/cli/tools/repl/mod.rs index 24bc8e30a5a104..a30304687983d8 100644 --- a/cli/tools/repl/mod.rs +++ b/cli/tools/repl/mod.rs @@ -162,7 +162,7 @@ pub async fn run( let factory = CliFactory::from_flags(flags); let cli_options = factory.cli_options()?; let main_module = cli_options.resolve_main_module()?; - let permissions = factory.create_permissions_container()?; + let permissions = factory.root_permissions_container()?; let npm_resolver = factory.npm_resolver().await?.clone(); let resolver = factory.resolver().await?.clone(); let file_fetcher = factory.file_fetcher()?; @@ -177,7 +177,7 @@ pub async fn run( .create_custom_worker( WorkerExecutionMode::Repl, main_module.clone(), - permissions, + permissions.clone(), vec![crate::ops::testing::deno_test::init_ops(test_event_sender)], Default::default(), ) @@ -189,7 +189,7 @@ pub async fn run( npm_resolver, resolver, worker, - main_module, + main_module.clone(), test_event_receiver, ) .await?; diff --git a/cli/tools/repl/session.rs b/cli/tools/repl/session.rs index 484664dae493a1..26e1eeac2f851a 100644 --- a/cli/tools/repl/session.rs +++ b/cli/tools/repl/session.rs @@ -7,7 +7,7 @@ use crate::cdp; use crate::colors; use crate::lsp::ReplLanguageServer; use crate::npm::CliNpmResolver; -use crate::resolver::CliGraphResolver; +use crate::resolver::CliResolver; use crate::tools::test::report_tests; use crate::tools::test::reporters::PrettyTestReporter; use crate::tools::test::reporters::TestReporter; @@ -25,6 +25,7 @@ use deno_ast::swc::visit::noop_visit_type; use deno_ast::swc::visit::Visit; use deno_ast::swc::visit::VisitWith; use deno_ast::ImportsNotUsedAsValues; +use deno_ast::ModuleKind; use deno_ast::ModuleSpecifier; use deno_ast::ParseDiagnosticsError; use deno_ast::ParsedSource; @@ -42,13 +43,13 @@ use deno_core::unsync::spawn; use deno_core::url::Url; use deno_core::LocalInspectorSession; use deno_core::PollEventLoopOptions; -use deno_graph::source::ResolutionMode; -use deno_graph::source::Resolver; use deno_graph::Position; use deno_graph::PositionRange; use deno_graph::SpecifierWithRange; use deno_runtime::worker::MainWorker; use deno_semver::npm::NpmPackageReqReference; +use node_resolver::NodeResolutionKind; +use node_resolver::ResolutionMode; use once_cell::sync::Lazy; use regex::Match; use regex::Regex; @@ -179,7 +180,7 @@ struct ReplJsxState { pub struct ReplSession { npm_resolver: Arc, - resolver: Arc, + resolver: Arc, pub worker: MainWorker, session: LocalInspectorSession, pub context_id: u64, @@ -198,7 +199,7 @@ impl ReplSession { pub async fn initialize( cli_options: &CliOptions, npm_resolver: Arc, - resolver: Arc, + resolver: Arc, mut worker: MainWorker, main_module: ModuleSpecifier, test_event_receiver: TestEventReceiver, @@ -244,7 +245,7 @@ impl ReplSession { assert_ne!(context_id, 0); let referrer = - deno_core::resolve_path("./$deno$repl.ts", cli_options.initial_cwd()) + deno_core::resolve_path("./$deno$repl.mts", cli_options.initial_cwd()) .unwrap(); let cwd_url = @@ -641,6 +642,10 @@ impl ReplSession { jsx_fragment_factory: self.jsx.frag_factory.clone(), jsx_import_source: self.jsx.import_source.clone(), var_decl_imports: true, + verbatim_module_syntax: false, + }, + &deno_ast::TranspileModuleOptions { + module_kind: Some(ModuleKind::Esm), }, &deno_ast::EmitOptions { source_map: deno_ast::SourceMapOption::None, @@ -651,7 +656,6 @@ impl ReplSession { }, )? .into_source() - .into_string()? .text; let value = self @@ -697,18 +701,19 @@ impl ReplSession { let mut collector = ImportCollector::new(); program.visit_with(&mut collector); - let referrer_range = deno_graph::Range { - specifier: self.referrer.clone(), - start: deno_graph::Position::zeroed(), - end: deno_graph::Position::zeroed(), - }; let resolved_imports = collector .imports .iter() .flat_map(|i| { self .resolver - .resolve(i, &referrer_range, ResolutionMode::Execution) + .resolve( + i, + &self.referrer, + deno_graph::Position::zeroed(), + ResolutionMode::Import, + NodeResolutionKind::Execution, + ) .ok() .or_else(|| ModuleSpecifier::parse(i).ok()) }) diff --git a/cli/tools/run/hmr.rs b/cli/tools/run/hmr.rs index 6ccf8e344bc581..373c207d6991ee 100644 --- a/cli/tools/run/hmr.rs +++ b/cli/tools/run/hmr.rs @@ -1,9 +1,9 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use crate::cdp; -use crate::emit::Emitter; -use crate::util::file_watcher::WatcherCommunicator; -use crate::util::file_watcher::WatcherRestartMode; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; + use deno_core::error::generic_error; use deno_core::error::AnyError; use deno_core::futures::StreamExt; @@ -12,11 +12,13 @@ use deno_core::serde_json::{self}; use deno_core::url::Url; use deno_core::LocalInspectorSession; use deno_terminal::colors; -use std::collections::HashMap; -use std::path::PathBuf; -use std::sync::Arc; use tokio::select; +use crate::cdp; +use crate::emit::Emitter; +use crate::util::file_watcher::WatcherCommunicator; +use crate::util::file_watcher::WatcherRestartMode; + fn explain(status: &cdp::Status) -> &'static str { match status { cdp::Status::Ok => "OK", @@ -139,7 +141,7 @@ impl crate::worker::HmrRunner for HmrRunner { }; let source_code = self.emitter.load_and_emit_for_hmr( - &module_url + &module_url, ).await?; let mut tries = 1; diff --git a/cli/tools/run/mod.rs b/cli/tools/run/mod.rs index 2006444904380b..8fab544ecacae2 100644 --- a/cli/tools/run/mod.rs +++ b/cli/tools/run/mod.rs @@ -30,6 +30,16 @@ To grant permissions, set them before the script argument. For example: } } +fn set_npm_user_agent() { + static ONCE: std::sync::Once = std::sync::Once::new(); + ONCE.call_once(|| { + std::env::set_var( + crate::npm::NPM_CONFIG_USER_AGENT_ENV_VAR, + crate::npm::get_npm_config_user_agent(), + ); + }); +} + pub async fn run_script( mode: WorkerExecutionMode, flags: Arc, @@ -58,12 +68,15 @@ pub async fn run_script( let main_module = cli_options.resolve_main_module()?; + if main_module.scheme() == "npm" { + set_npm_user_agent(); + } + maybe_npm_install(&factory).await?; - let permissions = factory.create_permissions_container()?; let worker_factory = factory.create_cli_main_worker_factory().await?; let mut worker = worker_factory - .create_main_worker(mode, main_module, permissions) + .create_main_worker(mode, main_module.clone()) .await?; let exit_code = worker.run().await?; @@ -79,7 +92,6 @@ pub async fn run_from_stdin(flags: Arc) -> Result { let file_fetcher = factory.file_fetcher()?; let worker_factory = factory.create_cli_main_worker_factory().await?; - let permissions = factory.create_permissions_container()?; let mut source = Vec::new(); std::io::stdin().read_to_end(&mut source)?; // Save a fake file into file fetcher cache @@ -91,7 +103,7 @@ pub async fn run_from_stdin(flags: Arc) -> Result { }); let mut worker = worker_factory - .create_main_worker(WorkerExecutionMode::Run, main_module, permissions) + .create_main_worker(WorkerExecutionMode::Run, main_module.clone()) .await?; let exit_code = worker.run().await?; Ok(exit_code) @@ -112,7 +124,8 @@ async fn run_with_watch( !watch_flags.no_clear_screen, ), WatcherRestartMode::Automatic, - move |flags, watcher_communicator, _changed_paths| { + move |flags, watcher_communicator, changed_paths| { + watcher_communicator.show_path_changed(changed_paths.clone()); Ok(async move { let factory = CliFactory::from_flags_for_watcher( flags, @@ -121,15 +134,18 @@ async fn run_with_watch( let cli_options = factory.cli_options()?; let main_module = cli_options.resolve_main_module()?; + if main_module.scheme() == "npm" { + set_npm_user_agent(); + } + maybe_npm_install(&factory).await?; let _ = watcher_communicator.watch_paths(cli_options.watch_paths()); - let permissions = factory.create_permissions_container()?; let mut worker = factory .create_cli_main_worker_factory() .await? - .create_main_worker(mode, main_module, permissions) + .create_main_worker(mode, main_module.clone()) .await?; if watch_flags.hmr { @@ -173,10 +189,9 @@ pub async fn eval_command( source: source_code.into_bytes().into(), }); - let permissions = factory.create_permissions_container()?; let worker_factory = factory.create_cli_main_worker_factory().await?; let mut worker = worker_factory - .create_main_worker(WorkerExecutionMode::Eval, main_module, permissions) + .create_main_worker(WorkerExecutionMode::Eval, main_module.clone()) .await?; let exit_code = worker.run().await?; Ok(exit_code) diff --git a/cli/tools/serve.rs b/cli/tools/serve.rs index 2f553cf1ed0657..d7989140aeb20f 100644 --- a/cli/tools/serve.rs +++ b/cli/tools/serve.rs @@ -5,7 +5,6 @@ use std::sync::Arc; use deno_core::error::AnyError; use deno_core::futures::TryFutureExt; use deno_core::ModuleSpecifier; -use deno_runtime::deno_permissions::PermissionsContainer; use super::run::check_permission_before_script; use super::run::maybe_npm_install; @@ -44,15 +43,16 @@ pub async fn serve( maybe_npm_install(&factory).await?; - let permissions = factory.create_permissions_container()?; let worker_factory = factory.create_cli_main_worker_factory().await?; - + let hmr = serve_flags + .watch + .map(|watch_flags| watch_flags.hmr) + .unwrap_or(false); do_serve( worker_factory, - main_module, - permissions, + main_module.clone(), serve_flags.worker_count, - false, + hmr, ) .await } @@ -60,7 +60,6 @@ pub async fn serve( async fn do_serve( worker_factory: CliMainWorkerFactory, main_module: ModuleSpecifier, - permissions: PermissionsContainer, worker_count: Option, hmr: bool, ) -> Result { @@ -71,7 +70,6 @@ async fn do_serve( worker_count, }, main_module.clone(), - permissions.clone(), ) .await?; let worker_count = match worker_count { @@ -87,15 +85,13 @@ async fn do_serve( for i in 0..extra_workers { let worker_factory = worker_factory.clone(); let main_module = main_module.clone(); - let permissions = permissions.clone(); let (tx, rx) = tokio::sync::oneshot::channel(); channels.push(rx); std::thread::Builder::new() .name(format!("serve-worker-{i}")) .spawn(move || { deno_runtime::tokio_util::create_and_run_current_thread(async move { - let result = - run_worker(i, worker_factory, main_module, permissions, hmr).await; + let result = run_worker(i, worker_factory, main_module, hmr).await; let _ = tx.send(result); }); })?; @@ -116,25 +112,21 @@ async fn do_serve( } } Ok(exit_code) - - // main.await? } async fn run_worker( worker_count: usize, worker_factory: CliMainWorkerFactory, main_module: ModuleSpecifier, - permissions: PermissionsContainer, hmr: bool, ) -> Result { - let mut worker = worker_factory + let mut worker: crate::worker::CliMainWorker = worker_factory .create_main_worker( deno_runtime::WorkerExecutionMode::Serve { is_main: false, worker_count: Some(worker_count), }, main_module, - permissions, ) .await?; if hmr { @@ -159,7 +151,8 @@ async fn serve_with_watch( !watch_flags.no_clear_screen, ), WatcherRestartMode::Automatic, - move |flags, watcher_communicator, _changed_paths| { + move |flags, watcher_communicator, changed_paths| { + watcher_communicator.show_path_changed(changed_paths.clone()); Ok(async move { let factory = CliFactory::from_flags_for_watcher( flags, @@ -171,11 +164,9 @@ async fn serve_with_watch( maybe_npm_install(&factory).await?; let _ = watcher_communicator.watch_paths(cli_options.watch_paths()); - - let permissions = factory.create_permissions_container()?; let worker_factory = factory.create_cli_main_worker_factory().await?; - do_serve(worker_factory, main_module, permissions, worker_count, hmr) + do_serve(worker_factory, main_module.clone(), worker_count, hmr) .await?; Ok(()) diff --git a/cli/tools/task.rs b/cli/tools/task.rs index ae91f53a7ac10a..4752738c527a8c 100644 --- a/cli/tools/task.rs +++ b/cli/tools/task.rs @@ -1,23 +1,34 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use std::borrow::Cow; use std::collections::HashMap; use std::collections::HashSet; +use std::num::NonZeroUsize; use std::path::Path; use std::path::PathBuf; use std::rc::Rc; use std::sync::Arc; -use deno_config::deno_json::Task; +use console_static_text::ansi::strip_ansi_codes; +use deno_config::workspace::FolderConfigs; +use deno_config::workspace::TaskDefinition; use deno_config::workspace::TaskOrScript; use deno_config::workspace::WorkspaceDirectory; +use deno_config::workspace::WorkspaceMemberTasksConfig; use deno_config::workspace::WorkspaceTasksConfig; use deno_core::anyhow::anyhow; use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; -use deno_core::normalize_path; +use deno_core::futures::future::LocalBoxFuture; +use deno_core::futures::stream::futures_unordered; +use deno_core::futures::FutureExt; +use deno_core::futures::StreamExt; +use deno_core::url::Url; +use deno_path_util::normalize_path; +use deno_runtime::deno_node::NodeResolver; use deno_task_shell::ShellCommand; +use indexmap::IndexMap; +use regex::Regex; use crate::args::CliOptions; use crate::args::Flags; @@ -28,6 +39,12 @@ use crate::npm::CliNpmResolver; use crate::task_runner; use crate::util::fs::canonicalize_path; +#[derive(Debug)] +struct PackageTaskInfo { + matched_tasks: Vec, + tasks_config: WorkspaceTasksConfig, +} + pub async fn execute_script( flags: Arc, task_flags: TaskFlags, @@ -35,7 +52,7 @@ pub async fn execute_script( let factory = CliFactory::from_flags(flags); let cli_options = factory.cli_options()?; let start_dir = &cli_options.start_dir; - if !start_dir.has_deno_or_pkg_json() { + if !start_dir.has_deno_or_pkg_json() && !task_flags.eval { bail!("deno task couldn't find deno.json(c). See https://docs.deno.com/go/config") } let force_use_pkg_json = @@ -48,153 +65,563 @@ pub async fn execute_script( v == "1" }) .unwrap_or(false); - let tasks_config = start_dir.to_tasks_config()?; - let tasks_config = if force_use_pkg_json { - tasks_config.with_only_pkg_json() + + fn arg_to_regex(input: &str) -> Result { + let mut regex_str = regex::escape(input); + regex_str = regex_str.replace("\\*", ".*"); + + Regex::new(®ex_str) + } + + let packages_task_configs: Vec = if let Some(filter) = + &task_flags.filter + { + let task_name = task_flags.task.as_ref().unwrap(); + + // Filter based on package name + let package_regex = arg_to_regex(filter)?; + let task_regex = arg_to_regex(task_name)?; + + let mut packages_task_info: Vec = vec![]; + + fn matches_package( + config: &FolderConfigs, + force_use_pkg_json: bool, + regex: &Regex, + ) -> bool { + if !force_use_pkg_json { + if let Some(deno_json) = &config.deno_json { + if let Some(name) = &deno_json.json.name { + if regex.is_match(name) { + return true; + } + } + } + } + + if let Some(package_json) = &config.pkg_json { + if let Some(name) = &package_json.name { + if regex.is_match(name) { + return true; + } + } + } + + false + } + + let workspace = cli_options.workspace(); + for folder in workspace.config_folders() { + if !matches_package(folder.1, force_use_pkg_json, &package_regex) { + continue; + } + + let member_dir = workspace.resolve_member_dir(folder.0); + let mut tasks_config = member_dir.to_tasks_config()?; + if force_use_pkg_json { + tasks_config = tasks_config.with_only_pkg_json(); + } + + // Any of the matched tasks could be a child task of another matched + // one. Therefore we need to filter these out to ensure that every + // task is only run once. + let mut matched: HashSet = HashSet::new(); + let mut visited: HashSet = HashSet::new(); + + fn visit_task( + tasks_config: &WorkspaceTasksConfig, + visited: &mut HashSet, + name: &str, + ) { + if visited.contains(name) { + return; + } + + visited.insert(name.to_string()); + + if let Some((_, TaskOrScript::Task(_, task))) = &tasks_config.task(name) + { + for dep in &task.dependencies { + visit_task(tasks_config, visited, dep); + } + } + } + + // Match tasks in deno.json + for name in tasks_config.task_names() { + if task_regex.is_match(name) && !visited.contains(name) { + matched.insert(name.to_string()); + visit_task(&tasks_config, &mut visited, name); + } + } + + packages_task_info.push(PackageTaskInfo { + matched_tasks: matched + .iter() + .map(|s| s.to_string()) + .collect::>(), + tasks_config, + }); + } + + // Logging every task definition would be too spammy. Pnpm only + // logs a simple message too. + if packages_task_info + .iter() + .all(|config| config.matched_tasks.is_empty()) + { + log::warn!( + "{}", + colors::red(format!( + "No matching task or script '{}' found in selected packages.", + task_name + )) + ); + return Ok(0); + } + + // FIXME: Sort packages topologically + // + + packages_task_info } else { - tasks_config - }; + let mut tasks_config = start_dir.to_tasks_config()?; - let task_name = match &task_flags.task { - Some(task) => task, - None => { + if force_use_pkg_json { + tasks_config = tasks_config.with_only_pkg_json() + } + + let Some(task_name) = &task_flags.task else { print_available_tasks( &mut std::io::stdout(), &cli_options.start_dir, &tasks_config, )?; return Ok(0); - } + }; + + vec![PackageTaskInfo { + tasks_config, + matched_tasks: vec![task_name.to_string()], + }] }; let npm_resolver = factory.npm_resolver().await?; let node_resolver = factory.node_resolver().await?; let env_vars = task_runner::real_env_vars(); - match tasks_config.task(task_name) { - Some((dir_url, task_or_script)) => match task_or_script { - TaskOrScript::Task(_tasks, script) => { - let cwd = match task_flags.cwd { - Some(path) => canonicalize_path(&PathBuf::from(path)) - .context("failed canonicalizing --cwd")?, - None => normalize_path(dir_url.to_file_path().unwrap()), - }; - - let custom_commands = task_runner::resolve_custom_commands( - npm_resolver.as_ref(), - node_resolver, - )?; - run_task(RunTaskOptions { - task_name, - script, - cwd: &cwd, - env_vars, - custom_commands, - npm_resolver: npm_resolver.as_ref(), - cli_options, - }) - .await - } - TaskOrScript::Script(scripts, _script) => { - // ensure the npm packages are installed if using a managed resolver - if let Some(npm_resolver) = npm_resolver.as_managed() { - npm_resolver.ensure_top_level_package_json_install().await?; - } + let no_of_concurrent_tasks = if let Ok(value) = std::env::var("DENO_JOBS") { + value.parse::().ok() + } else { + std::thread::available_parallelism().ok() + } + .unwrap_or_else(|| NonZeroUsize::new(2).unwrap()); - let cwd = match task_flags.cwd { - Some(path) => canonicalize_path(&PathBuf::from(path))?, - None => normalize_path(dir_url.to_file_path().unwrap()), - }; - - // At this point we already checked if the task name exists in package.json. - // We can therefore check for "pre" and "post" scripts too, since we're only - // dealing with package.json here and not deno.json - let task_names = vec![ - format!("pre{}", task_name), - task_name.clone(), - format!("post{}", task_name), - ]; - let custom_commands = task_runner::resolve_custom_commands( - npm_resolver.as_ref(), - node_resolver, - )?; - for task_name in &task_names { - if let Some(script) = scripts.get(task_name) { - let exit_code = run_task(RunTaskOptions { - task_name, - script, - cwd: &cwd, - env_vars: env_vars.clone(), - custom_commands: custom_commands.clone(), - npm_resolver: npm_resolver.as_ref(), - cli_options, - }) - .await?; - if exit_code > 0 { - return Ok(exit_code); - } - } - } + let task_runner = TaskRunner { + task_flags: &task_flags, + npm_resolver: npm_resolver.as_ref(), + node_resolver: node_resolver.as_ref(), + env_vars, + cli_options, + concurrency: no_of_concurrent_tasks.into(), + }; - Ok(0) - } - }, - None => { - if task_flags.is_run { - return Err(anyhow!("Task not found: {}", task_name)); - } - log::error!("Task not found: {}", task_name); - if log::log_enabled!(log::Level::Error) { - print_available_tasks( - &mut std::io::stderr(), - &cli_options.start_dir, - &tasks_config, - )?; - } - Ok(1) + if task_flags.eval { + return task_runner + .run_deno_task( + &Url::from_directory_path(cli_options.initial_cwd()).unwrap(), + "", + &TaskDefinition { + command: task_flags.task.as_ref().unwrap().to_string(), + dependencies: vec![], + description: None, + }, + ) + .await; + } + + for task_config in &packages_task_configs { + let exit_code = task_runner.run_tasks(task_config).await?; + if exit_code > 0 { + return Ok(exit_code); } } + + Ok(0) } -struct RunTaskOptions<'a> { +struct RunSingleOptions<'a> { task_name: &'a str, script: &'a str, cwd: &'a Path, - env_vars: HashMap, custom_commands: HashMap>, +} + +struct TaskRunner<'a> { + task_flags: &'a TaskFlags, npm_resolver: &'a dyn CliNpmResolver, + node_resolver: &'a NodeResolver, + env_vars: HashMap, cli_options: &'a CliOptions, + concurrency: usize, } -async fn run_task(opts: RunTaskOptions<'_>) -> Result { - let RunTaskOptions { - task_name, - script, - cwd, - env_vars, - custom_commands, - npm_resolver, - cli_options, - } = opts; +impl<'a> TaskRunner<'a> { + pub async fn run_tasks( + &self, + pkg_tasks_config: &PackageTaskInfo, + ) -> Result { + match sort_tasks_topo(pkg_tasks_config) { + Ok(sorted) => self.run_tasks_in_parallel(sorted).await, + Err(err) => match err { + TaskError::NotFound(name) => { + if self.task_flags.is_run { + return Err(anyhow!("Task not found: {}", name)); + } - output_task( - opts.task_name, - &task_runner::get_script_with_args(script, cli_options.argv()), - ); + log::error!("Task not found: {}", name); + if log::log_enabled!(log::Level::Error) { + self.print_available_tasks(&pkg_tasks_config.tasks_config)?; + } + Ok(1) + } + TaskError::TaskDepCycle { path } => { + log::error!("Task cycle detected: {}", path.join(" -> ")); + Ok(1) + } + }, + } + } - task_runner::run_task(task_runner::RunTaskOptions { - task_name, - script, - cwd, - env_vars, - custom_commands, - init_cwd: opts.cli_options.initial_cwd(), - argv: cli_options.argv(), - root_node_modules_dir: npm_resolver - .root_node_modules_path() - .map(|p| p.as_path()), - }) - .await + pub fn print_available_tasks( + &self, + tasks_config: &WorkspaceTasksConfig, + ) -> Result<(), std::io::Error> { + print_available_tasks( + &mut std::io::stderr(), + &self.cli_options.start_dir, + tasks_config, + ) + } + + async fn run_tasks_in_parallel( + &self, + tasks: Vec>, + ) -> Result { + struct PendingTasksContext<'a> { + completed: HashSet, + running: HashSet, + tasks: &'a [ResolvedTask<'a>], + } + + impl<'a> PendingTasksContext<'a> { + fn has_remaining_tasks(&self) -> bool { + self.completed.len() < self.tasks.len() + } + + fn mark_complete(&mut self, task: &ResolvedTask) { + self.running.remove(&task.id); + self.completed.insert(task.id); + } + + fn get_next_task<'b>( + &mut self, + runner: &'b TaskRunner<'b>, + ) -> Option< + LocalBoxFuture<'b, Result<(i32, &'a ResolvedTask<'a>), AnyError>>, + > + where + 'a: 'b, + { + for task in self.tasks.iter() { + if self.completed.contains(&task.id) + || self.running.contains(&task.id) + { + continue; + } + + let should_run = task + .dependencies + .iter() + .all(|dep_id| self.completed.contains(dep_id)); + if !should_run { + continue; + } + + self.running.insert(task.id); + return Some( + async move { + match task.task_or_script { + TaskOrScript::Task(_, def) => { + runner.run_deno_task(task.folder_url, task.name, def).await + } + TaskOrScript::Script(scripts, _) => { + runner + .run_npm_script(task.folder_url, task.name, scripts) + .await + } + } + .map(|exit_code| (exit_code, task)) + } + .boxed_local(), + ); + } + None + } + } + + let mut context = PendingTasksContext { + completed: HashSet::with_capacity(tasks.len()), + running: HashSet::with_capacity(self.concurrency), + tasks: &tasks, + }; + + let mut queue = futures_unordered::FuturesUnordered::new(); + + while context.has_remaining_tasks() { + while queue.len() < self.concurrency { + if let Some(task) = context.get_next_task(self) { + queue.push(task); + } else { + break; + } + } + + // If queue is empty at this point, then there are no more tasks in the queue. + let Some(result) = queue.next().await else { + debug_assert_eq!(context.tasks.len(), 0); + break; + }; + + let (exit_code, name) = result?; + if exit_code > 0 { + return Ok(exit_code); + } + + context.mark_complete(name); + } + + Ok(0) + } + + pub async fn run_deno_task( + &self, + dir_url: &Url, + task_name: &str, + definition: &TaskDefinition, + ) -> Result { + let cwd = match &self.task_flags.cwd { + Some(path) => canonicalize_path(&PathBuf::from(path)) + .context("failed canonicalizing --cwd")?, + None => normalize_path(dir_url.to_file_path().unwrap()), + }; + + let custom_commands = task_runner::resolve_custom_commands( + self.npm_resolver, + self.node_resolver, + )?; + self + .run_single(RunSingleOptions { + task_name, + script: &definition.command, + cwd: &cwd, + custom_commands, + }) + .await + } + + pub async fn run_npm_script( + &self, + dir_url: &Url, + task_name: &str, + scripts: &IndexMap, + ) -> Result { + // ensure the npm packages are installed if using a managed resolver + if let Some(npm_resolver) = self.npm_resolver.as_managed() { + npm_resolver.ensure_top_level_package_json_install().await?; + } + + let cwd = match &self.task_flags.cwd { + Some(path) => canonicalize_path(&PathBuf::from(path))?, + None => normalize_path(dir_url.to_file_path().unwrap()), + }; + + // At this point we already checked if the task name exists in package.json. + // We can therefore check for "pre" and "post" scripts too, since we're only + // dealing with package.json here and not deno.json + let task_names = vec![ + format!("pre{}", task_name), + task_name.to_string(), + format!("post{}", task_name), + ]; + let custom_commands = task_runner::resolve_custom_commands( + self.npm_resolver, + self.node_resolver, + )?; + for task_name in &task_names { + if let Some(script) = scripts.get(task_name) { + let exit_code = self + .run_single(RunSingleOptions { + task_name, + script, + cwd: &cwd, + custom_commands: custom_commands.clone(), + }) + .await?; + if exit_code > 0 { + return Ok(exit_code); + } + } + } + + Ok(0) + } + + async fn run_single( + &self, + opts: RunSingleOptions<'_>, + ) -> Result { + let RunSingleOptions { + task_name, + script, + cwd, + custom_commands, + } = opts; + + output_task( + opts.task_name, + &task_runner::get_script_with_args(script, self.cli_options.argv()), + ); + + Ok( + task_runner::run_task(task_runner::RunTaskOptions { + task_name, + script, + cwd, + env_vars: self.env_vars.clone(), + custom_commands, + init_cwd: self.cli_options.initial_cwd(), + argv: self.cli_options.argv(), + root_node_modules_dir: self.npm_resolver.root_node_modules_path(), + stdio: None, + }) + .await? + .exit_code, + ) + } +} + +#[derive(Debug)] +enum TaskError { + NotFound(String), + TaskDepCycle { path: Vec }, +} + +struct ResolvedTask<'a> { + id: usize, + name: &'a str, + folder_url: &'a Url, + task_or_script: TaskOrScript<'a>, + dependencies: Vec, +} + +fn sort_tasks_topo<'a>( + pkg_task_config: &'a PackageTaskInfo, +) -> Result>, TaskError> { + trait TasksConfig { + fn task( + &self, + name: &str, + ) -> Option<(&Url, TaskOrScript, &dyn TasksConfig)>; + } + + impl TasksConfig for WorkspaceTasksConfig { + fn task( + &self, + name: &str, + ) -> Option<(&Url, TaskOrScript, &dyn TasksConfig)> { + if let Some(member) = &self.member { + if let Some((dir_url, task_or_script)) = member.task(name) { + return Some((dir_url, task_or_script, self as &dyn TasksConfig)); + } + } + if let Some(root) = &self.root { + if let Some((dir_url, task_or_script)) = root.task(name) { + // switch to only using the root tasks for the dependencies + return Some((dir_url, task_or_script, root as &dyn TasksConfig)); + } + } + None + } + } + + impl TasksConfig for WorkspaceMemberTasksConfig { + fn task( + &self, + name: &str, + ) -> Option<(&Url, TaskOrScript, &dyn TasksConfig)> { + self.task(name).map(|(dir_url, task_or_script)| { + (dir_url, task_or_script, self as &dyn TasksConfig) + }) + } + } + + fn sort_visit<'a>( + name: &'a str, + sorted: &mut Vec>, + mut path: Vec<(&'a Url, &'a str)>, + tasks_config: &'a dyn TasksConfig, + ) -> Result { + let Some((folder_url, task_or_script, tasks_config)) = + tasks_config.task(name) + else { + return Err(TaskError::NotFound(name.to_string())); + }; + + if let Some(existing_task) = sorted + .iter() + .find(|task| task.name == name && task.folder_url == folder_url) + { + // already exists + return Ok(existing_task.id); + } + + if path.contains(&(folder_url, name)) { + path.push((folder_url, name)); + return Err(TaskError::TaskDepCycle { + path: path.iter().map(|(_, s)| s.to_string()).collect(), + }); + } + + let mut dependencies: Vec = Vec::new(); + if let TaskOrScript::Task(_, task) = task_or_script { + dependencies.reserve(task.dependencies.len()); + for dep in &task.dependencies { + let mut path = path.clone(); + path.push((folder_url, name)); + dependencies.push(sort_visit(dep, sorted, path, tasks_config)?); + } + } + + let id = sorted.len(); + sorted.push(ResolvedTask { + id, + name, + folder_url, + task_or_script, + dependencies, + }); + + Ok(id) + } + + let mut sorted: Vec> = vec![]; + + for name in &pkg_task_config.matched_tasks { + sort_visit(name, &mut sorted, Vec::new(), &pkg_task_config.tasks_config)?; + } + + Ok(sorted) } fn output_task(task_name: &str, script: &str) { @@ -220,80 +647,123 @@ fn print_available_tasks( " {}", colors::red("No tasks found in configuration file") )?; - } else { - let mut seen_task_names = - HashSet::with_capacity(tasks_config.tasks_count()); - for maybe_config in [&tasks_config.member, &tasks_config.root] { - let Some(config) = maybe_config else { - continue; - }; - for (is_root, is_deno, (key, task)) in config - .deno_json - .as_ref() - .map(|config| { - let is_root = !is_cwd_root_dir - && config.folder_url - == *workspace_dir.workspace.root_dir().as_ref(); - config - .tasks - .iter() - .map(move |(k, t)| (is_root, true, (k, Cow::Borrowed(t)))) - }) - .into_iter() - .flatten() - .chain( - config - .package_json - .as_ref() - .map(|config| { - let is_root = !is_cwd_root_dir - && config.folder_url - == *workspace_dir.workspace.root_dir().as_ref(); - config.tasks.iter().map(move |(k, v)| { - (is_root, false, (k, Cow::Owned(Task::Definition(v.clone())))) - }) - }) - .into_iter() - .flatten(), - ) - { - if !seen_task_names.insert(key) { + return Ok(()); + } + + struct AvailableTaskDescription { + is_root: bool, + is_deno: bool, + name: String, + task: TaskDefinition, + } + let mut seen_task_names = HashSet::with_capacity(tasks_config.tasks_count()); + let mut task_descriptions = Vec::with_capacity(tasks_config.tasks_count()); + + for maybe_config in [&tasks_config.member, &tasks_config.root] { + let Some(config) = maybe_config else { + continue; + }; + + if let Some(config) = config.deno_json.as_ref() { + let is_root = !is_cwd_root_dir + && config.folder_url == *workspace_dir.workspace.root_dir().as_ref(); + + for (name, definition) in &config.tasks { + if !seen_task_names.insert(name) { continue; // already seen } + task_descriptions.push(AvailableTaskDescription { + is_root, + is_deno: true, + name: name.to_string(), + task: definition.clone(), + }); + } + } + + if let Some(config) = config.package_json.as_ref() { + let is_root = !is_cwd_root_dir + && config.folder_url == *workspace_dir.workspace.root_dir().as_ref(); + for (name, script) in &config.tasks { + if !seen_task_names.insert(name) { + continue; // already seen + } + + task_descriptions.push(AvailableTaskDescription { + is_root, + is_deno: false, + name: name.to_string(), + task: deno_config::deno_json::TaskDefinition { + command: script.to_string(), + dependencies: vec![], + description: None, + }, + }); + } + } + } + + for desc in task_descriptions { + writeln!( + writer, + "- {}{}", + colors::cyan(desc.name), + if desc.is_root { + if desc.is_deno { + format!(" {}", colors::italic_gray("(workspace)")) + } else { + format!(" {}", colors::italic_gray("(workspace package.json)")) + } + } else if desc.is_deno { + "".to_string() + } else { + format!(" {}", colors::italic_gray("(package.json)")) + } + )?; + if let Some(description) = &desc.task.description { + let slash_slash = colors::italic_gray("//"); + for line in description.lines() { writeln!( writer, - "- {}{}", - colors::cyan(key), - if is_root { - if is_deno { - format!(" {}", colors::italic_gray("(workspace)")) - } else { - format!(" {}", colors::italic_gray("(workspace package.json)")) - } - } else if is_deno { - "".to_string() - } else { - format!(" {}", colors::italic_gray("(package.json)")) - } + " {slash_slash} {}", + colors::italic_gray(strip_ansi_codes_and_escape_control_chars(line)) )?; - let definition = match task.as_ref() { - Task::Definition(definition) => definition, - Task::Commented { definition, .. } => definition, - }; - if let Task::Commented { comments, .. } = task.as_ref() { - let slash_slash = colors::italic_gray("//"); - for comment in comments { - writeln!( - writer, - " {slash_slash} {}", - colors::italic_gray(comment) - )?; - } - } - writeln!(writer, " {definition}")?; } } + writeln!( + writer, + " {}", + strip_ansi_codes_and_escape_control_chars(&desc.task.command) + )?; + if !desc.task.dependencies.is_empty() { + let dependencies = desc + .task + .dependencies + .into_iter() + .map(|d| strip_ansi_codes_and_escape_control_chars(&d)) + .collect::>() + .join(", "); + writeln!( + writer, + " {} {}", + colors::gray("depends on:"), + colors::cyan(dependencies) + )?; + } } Ok(()) } + +fn strip_ansi_codes_and_escape_control_chars(s: &str) -> String { + strip_ansi_codes(s) + .chars() + .map(|c| match c { + '\n' => "\\n".to_string(), + '\r' => "\\r".to_string(), + '\t' => "\\t".to_string(), + c if c.is_control() => format!("\\x{:02x}", c as u8), + c => c.to_string(), + }) + .collect() +} diff --git a/cli/tools/test/mod.rs b/cli/tools/test/mod.rs index e81abad0b26987..6357ebcae24a14 100644 --- a/cli/tools/test/mod.rs +++ b/cli/tools/test/mod.rs @@ -631,7 +631,7 @@ async fn configure_main_worker( "Deno[Deno.internal].core.setLeakTracingEnabled(true);", )?; } - let res = worker.execute_side_module_possibly_with_npm().await; + let res = worker.execute_side_module().await; let mut worker = worker.into_main_worker(); match res { Ok(()) => Ok(()), @@ -1357,6 +1357,7 @@ pub async fn report_tests( if let Err(err) = reporter.flush_report(&elapsed, &tests, &test_steps) { eprint!("Test reporter failed to flush: {}", err) } + #[allow(clippy::disallowed_methods)] std::process::exit(130); } } @@ -1642,6 +1643,7 @@ pub async fn run_tests_with_watch( loop { signal::ctrl_c().await.unwrap(); if !HAS_TEST_RUN_SIGINT_HANDLER.load(Ordering::Relaxed) { + #[allow(clippy::disallowed_methods)] std::process::exit(130); } } @@ -1659,6 +1661,7 @@ pub async fn run_tests_with_watch( ), move |flags, watcher_communicator, changed_paths| { let test_flags = test_flags.clone(); + watcher_communicator.show_path_changed(changed_paths.clone()); Ok(async move { let factory = CliFactory::from_flags_for_watcher( flags, diff --git a/cli/tools/upgrade.rs b/cli/tools/upgrade.rs index 7f21e6649c2c5b..cb85859f7ab2bd 100644 --- a/cli/tools/upgrade.rs +++ b/cli/tools/upgrade.rs @@ -540,7 +540,7 @@ pub async fn upgrade( let Some(archive_data) = download_package(&client, download_url).await? else { log::error!("Download could not be found, aborting"); - std::process::exit(1) + deno_runtime::exit(1) }; log::info!( @@ -579,6 +579,10 @@ pub async fn upgrade( let output_exe_path = full_path_output_flag.as_ref().unwrap_or(¤t_exe_path); + + #[cfg(windows)] + kill_running_deno_lsp_processes(); + let output_result = if *output_exe_path == current_exe_path { replace_exe(&new_exe_path, output_exe_path) } else { @@ -913,7 +917,7 @@ async fn download_package( // text above which will stay alive after the progress bars are complete let progress = progress_bar.update(""); let maybe_bytes = client - .download_with_progress(download_url.clone(), None, &progress) + .download_with_progress_and_retries(download_url.clone(), None, &progress) .await .with_context(|| format!("Failed downloading {download_url}. The version you requested may not have been built for the current architecture."))?; Ok(maybe_bytes) @@ -966,6 +970,34 @@ fn check_windows_access_denied_error( }) } +#[cfg(windows)] +fn kill_running_deno_lsp_processes() { + // limit this to `deno lsp` invocations to avoid killing important programs someone might be running + let is_debug = log::log_enabled!(log::Level::Debug); + let get_pipe = || { + if is_debug { + std::process::Stdio::inherit() + } else { + std::process::Stdio::null() + } + }; + let _ = Command::new("powershell.exe") + .args([ + "-Command", + r#"Get-WmiObject Win32_Process | Where-Object { + $_.Name -eq 'deno.exe' -and + $_.CommandLine -match '^(?:\"[^\"]+\"|\S+)\s+lsp\b' +} | ForEach-Object { + if ($_.Terminate()) { + Write-Host 'Terminated:' $_.ProcessId + } +}"#, + ]) + .stdout(get_pipe()) + .stderr(get_pipe()) + .output(); +} + fn set_exe_permissions( current_exe_path: &Path, output_exe_path: &Path, diff --git a/cli/tsc/99_main_compiler.js b/cli/tsc/99_main_compiler.js index 719f2b9824b052..7e8a407cf982a2 100644 --- a/cli/tsc/99_main_compiler.js +++ b/cli/tsc/99_main_compiler.js @@ -121,8 +121,8 @@ delete Object.prototype.__proto__; /** @type {Map} */ const sourceFileCache = new Map(); - /** @type {Map} */ - const sourceTextCache = new Map(); + /** @type {Map} */ + const scriptSnapshotCache = new Map(); /** @type {Map} */ const sourceRefCounts = new Map(); @@ -133,9 +133,6 @@ delete Object.prototype.__proto__; /** @type {Map} */ const isNodeSourceFileCache = new Map(); - /** @type {Map} */ - const isCjsCache = new Map(); - // Maps asset specifiers to the first scope that the asset was loaded into. /** @type {Map} */ const assetScopes = new Map(); @@ -210,12 +207,13 @@ delete Object.prototype.__proto__; const mapKey = path + key; let sourceFile = documentRegistrySourceFileCache.get(mapKey); if (!sourceFile || sourceFile.version !== version) { + const isCjs = /** @type {any} */ (scriptSnapshot).isCjs; sourceFile = ts.createLanguageServiceSourceFile( fileName, scriptSnapshot, { ...getCreateSourceFileOptions(sourceFileOptions), - impliedNodeFormat: (isCjsCache.get(fileName) ?? false) + impliedNodeFormat: isCjs ? ts.ModuleKind.CommonJS : ts.ModuleKind.ESNext, // in the lsp we want to be able to show documentation @@ -320,7 +318,7 @@ delete Object.prototype.__proto__; if (lastRequestMethod != "cleanupSemanticCache") { const mapKey = path + key; documentRegistrySourceFileCache.delete(mapKey); - sourceTextCache.delete(path); + scriptSnapshotCache.delete(path); ops.op_release(path); } } else { @@ -452,6 +450,12 @@ delete Object.prototype.__proto__; // We specify the resolution mode to be CommonJS for some npm files and this // diagnostic gets generated even though we're using custom module resolution. 1452, + // Module '...' cannot be imported using this construct. The specifier only resolves to an + // ES module, which cannot be imported with 'require'. + 1471, + // TS1479: The current file is a CommonJS module whose imports will produce 'require' calls; + // however, the referenced file is an ECMAScript module and cannot be imported with 'require'. + 1479, // TS2306: File '.../index.d.ts' is not a module. // We get this for `x-typescript-types` declaration files which don't export // anything. We prefer to treat these as modules with no exports. @@ -516,7 +520,6 @@ delete Object.prototype.__proto__; /** @typedef {{ * ls: ts.LanguageService & { [k:string]: any }, * compilerOptions: ts.CompilerOptions, - * forceEnabledVerbatimModuleSyntax: boolean, * }} LanguageServiceEntry */ /** @type {{ unscoped: LanguageServiceEntry, byScope: Map }} */ const languageServiceEntries = { @@ -625,8 +628,6 @@ delete Object.prototype.__proto__; `"data" is unexpectedly null for "${specifier}".`, ); - isCjsCache.set(specifier, isCjs); - sourceFile = ts.createSourceFile( specifier, data, @@ -680,14 +681,18 @@ delete Object.prototype.__proto__; getNewLine() { return "\n"; }, - resolveTypeReferenceDirectives( - typeDirectiveNames, + resolveTypeReferenceDirectiveReferences( + typeDirectiveReferences, containingFilePath, redirectedReference, options, - containingFileMode, + containingSourceFile, + _reusedNames, ) { - return typeDirectiveNames.map((arg) => { + const isCjs = + containingSourceFile?.impliedNodeFormat === ts.ModuleKind.CommonJS; + /** @type {Array} */ + const result = typeDirectiveReferences.map((arg) => { /** @type {ts.FileReference} */ const fileReference = typeof arg === "string" ? { @@ -700,16 +705,28 @@ delete Object.prototype.__proto__; /** @type {[string, ts.Extension] | undefined} */ const resolved = ops.op_resolve( containingFilePath, - isCjsCache.get(containingFilePath) ?? false, - [fileReference.fileName], + [ + [ + fileReference.resolutionMode == null + ? isCjs + : fileReference.resolutionMode === ts.ModuleKind.CommonJS, + fileReference.fileName, + ], + ], )?.[0]; if (resolved) { return { - primary: true, - resolvedFileName: resolved[0], + resolvedTypeReferenceDirective: { + primary: true, + resolvedFileName: resolved[0], + // todo(dsherret): we should probably be setting this + isExternalLibraryImport: undefined, + }, }; } else { - return undefined; + return { + resolvedTypeReferenceDirective: undefined, + }; } } else { return ts.resolveTypeReferenceDirective( @@ -719,34 +736,56 @@ delete Object.prototype.__proto__; host, redirectedReference, undefined, - containingFileMode ?? fileReference.resolutionMode, - ).resolvedTypeReferenceDirective; + containingSourceFile?.impliedNodeFormat ?? + fileReference.resolutionMode, + ); } }); + return result; }, - resolveModuleNames(specifiers, base) { + resolveModuleNameLiterals( + moduleLiterals, + base, + _redirectedReference, + compilerOptions, + containingSourceFile, + _reusedNames, + ) { + const specifiers = moduleLiterals.map((literal) => [ + ts.getModeForUsageLocation( + containingSourceFile, + literal, + compilerOptions, + ) === ts.ModuleKind.CommonJS, + literal.text, + ]); if (logDebug) { debug(`host.resolveModuleNames()`); debug(` base: ${base}`); - debug(` specifiers: ${specifiers.join(", ")}`); + debug(` specifiers: ${specifiers.map((s) => s[1]).join(", ")}`); } /** @type {Array<[string, ts.Extension] | undefined>} */ const resolved = ops.op_resolve( base, - isCjsCache.get(base) ?? false, specifiers, ); if (resolved) { + /** @type {Array} */ const result = resolved.map((item) => { if (item) { const [resolvedFileName, extension] = item; return { - resolvedFileName, - extension, - isExternalLibraryImport: false, + resolvedModule: { + resolvedFileName, + extension, + // todo(dsherret): we should probably be setting this + isExternalLibraryImport: false, + }, }; } - return undefined; + return { + resolvedModule: undefined, + }; }); result.length = specifiers.length; return result; @@ -802,27 +841,32 @@ delete Object.prototype.__proto__; if (logDebug) { debug(`host.getScriptSnapshot("${specifier}")`); } - const sourceFile = sourceFileCache.get(specifier); - if (sourceFile) { - if (!assetScopes.has(specifier)) { - assetScopes.set(specifier, lastRequestScope); + if (specifier.startsWith(ASSETS_URL_PREFIX)) { + const sourceFile = this.getSourceFile( + specifier, + ts.ScriptTarget.ESNext, + ); + if (sourceFile) { + if (!assetScopes.has(specifier)) { + assetScopes.set(specifier, lastRequestScope); + } + // This case only occurs for assets. + return ts.ScriptSnapshot.fromString(sourceFile.text); } - // This case only occurs for assets. - return ts.ScriptSnapshot.fromString(sourceFile.text); } - let sourceText = sourceTextCache.get(specifier); - if (sourceText == undefined) { + let scriptSnapshot = scriptSnapshotCache.get(specifier); + if (scriptSnapshot == undefined) { /** @type {{ data: string, version: string, isCjs: boolean }} */ const fileInfo = ops.op_load(specifier); if (!fileInfo) { return undefined; } - isCjsCache.set(specifier, fileInfo.isCjs); - sourceTextCache.set(specifier, fileInfo.data); + scriptSnapshot = ts.ScriptSnapshot.fromString(fileInfo.data); + scriptSnapshot.isCjs = fileInfo.isCjs; + scriptSnapshotCache.set(specifier, scriptSnapshot); scriptVersionCache.set(specifier, fileInfo.version); - sourceText = fileInfo.data; } - return ts.ScriptSnapshot.fromString(sourceText); + return scriptSnapshot; }, }; @@ -846,6 +890,8 @@ delete Object.prototype.__proto__; jqueryMessage, "Cannot_find_name_0_Do_you_need_to_install_type_definitions_for_jQuery_Try_npm_i_save_dev_types_Slash_2592": jqueryMessage, + "Module_0_was_resolved_to_1_but_allowArbitraryExtensions_is_not_set_6263": + "Module '{0}' was resolved to '{1}', but importing these modules is not supported.", }; })()); @@ -1026,7 +1072,7 @@ delete Object.prototype.__proto__; : ts.sortAndDeduplicateDiagnostics( checkFiles.map((s) => program.getSemanticDiagnostics(s)).flat(), )), - ].filter(filterMapDiagnostic.bind(null, false)); + ].filter(filterMapDiagnostic); // emit the tsbuildinfo file // @ts-ignore: emitBuildInfo is not exposed (https://github.com/microsoft/TypeScript/issues/49871) @@ -1041,28 +1087,11 @@ delete Object.prototype.__proto__; debug("<<< exec stop"); } - /** - * @param {boolean} isLsp - * @param {ts.Diagnostic} diagnostic - */ - function filterMapDiagnostic(isLsp, diagnostic) { + /** @param {ts.Diagnostic} diagnostic */ + function filterMapDiagnostic(diagnostic) { if (IGNORED_DIAGNOSTICS.includes(diagnostic.code)) { return false; } - if (isLsp) { - // TS1484: `...` is a type and must be imported using a type-only import when 'verbatimModuleSyntax' is enabled. - // We force-enable `verbatimModuleSyntax` in the LSP so the `type` - // modifier is used when auto-importing types. But we don't want this - // diagnostic unless it was explicitly enabled by the user. - if (diagnostic.code == 1484) { - const entry = (lastRequestScope - ? languageServiceEntries.byScope.get(lastRequestScope) - : null) ?? languageServiceEntries.unscoped; - if (entry.forceEnabledVerbatimModuleSyntax) { - return false; - } - } - } // make the diagnostic for using an `export =` in an es module a warning if (diagnostic.code === 1203) { diagnostic.category = ts.DiagnosticCategory.Warning; @@ -1159,12 +1188,10 @@ delete Object.prototype.__proto__; "strict": true, "target": "esnext", "useDefineForClassFields": true, - "verbatimModuleSyntax": true, "jsx": "react", "jsxFactory": "React.createElement", "jsxFragmentFactory": "React.Fragment", }), - forceEnabledVerbatimModuleSyntax: true, }; setLogDebug(enableDebugLogging, "TSLS"); debug("serverInit()"); @@ -1230,17 +1257,8 @@ delete Object.prototype.__proto__; const ls = oldEntry ? oldEntry.ls : ts.createLanguageService(host, documentRegistry); - let forceEnabledVerbatimModuleSyntax = false; - if (!config["verbatimModuleSyntax"]) { - config["verbatimModuleSyntax"] = true; - forceEnabledVerbatimModuleSyntax = true; - } const compilerOptions = lspTsConfigToCompilerOptions(config); - newByScope.set(scope, { - ls, - compilerOptions, - forceEnabledVerbatimModuleSyntax, - }); + newByScope.set(scope, { ls, compilerOptions }); languageServiceEntries.byScope.delete(scope); } for (const oldEntry of languageServiceEntries.byScope.values()) { @@ -1260,7 +1278,7 @@ delete Object.prototype.__proto__; closed = true; } scriptVersionCache.delete(script); - sourceTextCache.delete(script); + scriptSnapshotCache.delete(script); } if (newConfigsByScope || opened || closed) { @@ -1305,7 +1323,7 @@ delete Object.prototype.__proto__; ...ls.getSemanticDiagnostics(specifier), ...ls.getSuggestionDiagnostics(specifier), ...ls.getSyntacticDiagnostics(specifier), - ].filter(filterMapDiagnostic.bind(null, true))); + ].filter(filterMapDiagnostic)); } return respond(id, diagnosticMap); } catch (e) { @@ -1366,18 +1384,12 @@ delete Object.prototype.__proto__; "console", "Console", "ErrorConstructor", - "exports", "gc", "Global", "ImportMeta", "localStorage", - "module", - "NodeModule", - "NodeRequire", - "process", "queueMicrotask", "RequestInit", - "require", "ResponseInit", "sessionStorage", "setImmediate", diff --git a/cli/tsc/diagnostics.rs b/cli/tsc/diagnostics.rs index b0394ec177262a..d3795706eb7619 100644 --- a/cli/tsc/diagnostics.rs +++ b/cli/tsc/diagnostics.rs @@ -323,7 +323,7 @@ impl Diagnostics { // todo(dsherret): use a short lived cache to prevent parsing // source maps so often if let Ok(source_map) = - SourceMap::from_slice(&fast_check_module.source_map) + SourceMap::from_slice(fast_check_module.source_map.as_bytes()) { if let Some(start) = d.start.as_mut() { let maybe_token = source_map diff --git a/cli/tsc/dts/lib.deno.ns.d.ts b/cli/tsc/dts/lib.deno.ns.d.ts index 36592e10dc3c76..d9f66f11a7802d 100644 --- a/cli/tsc/dts/lib.deno.ns.d.ts +++ b/cli/tsc/dts/lib.deno.ns.d.ts @@ -556,14 +556,23 @@ declare namespace Deno { */ env?: "inherit" | boolean | string[]; - /** Specifies if the `sys` permission should be requested or revoked. - * If set to `"inherit"`, the current `sys` permission will be inherited. - * If set to `true`, the global `sys` permission will be requested. - * If set to `false`, the global `sys` permission will be revoked. + /** Specifies if the `ffi` permission should be requested or revoked. + * If set to `"inherit"`, the current `ffi` permission will be inherited. + * If set to `true`, the global `ffi` permission will be requested. + * If set to `false`, the global `ffi` permission will be revoked. * * @default {false} */ - sys?: "inherit" | boolean | string[]; + ffi?: "inherit" | boolean | Array; + + /** Specifies if the `import` permission should be requested or revoked. + * If set to `"inherit"` the current `import` permission will be inherited. + * If set to `true`, the global `import` permission will be requested. + * If set to `false`, the global `import` permission will be revoked. + * If set to `Array`, the `import` permissions will be requested with the + * specified domains. + */ + import?: "inherit" | boolean | Array; /** Specifies if the `net` permission should be requested or revoked. * if set to `"inherit"`, the current `net` permission will be inherited. @@ -638,15 +647,6 @@ declare namespace Deno { */ net?: "inherit" | boolean | string[]; - /** Specifies if the `ffi` permission should be requested or revoked. - * If set to `"inherit"`, the current `ffi` permission will be inherited. - * If set to `true`, the global `ffi` permission will be requested. - * If set to `false`, the global `ffi` permission will be revoked. - * - * @default {false} - */ - ffi?: "inherit" | boolean | Array; - /** Specifies if the `read` permission should be requested or revoked. * If set to `"inherit"`, the current `read` permission will be inherited. * If set to `true`, the global `read` permission will be requested. @@ -667,6 +667,15 @@ declare namespace Deno { */ run?: "inherit" | boolean | Array; + /** Specifies if the `sys` permission should be requested or revoked. + * If set to `"inherit"`, the current `sys` permission will be inherited. + * If set to `true`, the global `sys` permission will be requested. + * If set to `false`, the global `sys` permission will be revoked. + * + * @default {false} + */ + sys?: "inherit" | boolean | string[]; + /** Specifies if the `write` permission should be requested or revoked. * If set to `"inherit"`, the current `write` permission will be inherited. * If set to `true`, the global `write` permission will be requested. @@ -2962,6 +2971,10 @@ declare namespace Deno { * field from `stat` on Mac/BSD and `ftCreationTime` on Windows. This may * not be available on all platforms. */ birthtime: Date | null; + /** The last change time of the file. This corresponds to the `ctime` + * field from `stat` on Mac/BSD and `ChangeTime` on Windows. This may + * not be available on all platforms. */ + ctime: Date | null; /** ID of the device containing the file. */ dev: number; /** Inode number. @@ -2970,8 +2983,7 @@ declare namespace Deno { ino: number | null; /** The underlying raw `st_mode` bits that contain the standard Unix * permissions for this file/directory. - * - * _Linux/Mac OS only._ */ + */ mode: number | null; /** Number of hard links pointing to this file. * @@ -4523,7 +4535,7 @@ declare namespace Deno { /** The object that is returned from a {@linkcode Deno.upgradeWebSocket} * request. * - * @category Web Sockets */ + * @category WebSockets */ export interface WebSocketUpgrade { /** The response object that represents the HTTP response to the client, * which should be used to the {@linkcode RequestEvent} `.respondWith()` for @@ -4537,7 +4549,7 @@ declare namespace Deno { /** Options which can be set when performing a * {@linkcode Deno.upgradeWebSocket} upgrade of a {@linkcode Request} * - * @category Web Sockets */ + * @category WebSockets */ export interface UpgradeWebSocketOptions { /** Sets the `.protocol` property on the client side web socket to the * value provided here, which should be one of the strings specified in the @@ -4585,7 +4597,7 @@ declare namespace Deno { * This operation does not yet consume the request or open the websocket. This * only happens once the returned response has been passed to `respondWith()`. * - * @category Web Sockets + * @category WebSockets */ export function upgradeWebSocket( request: Request, diff --git a/cli/tsc/dts/lib.deno.shared_globals.d.ts b/cli/tsc/dts/lib.deno.shared_globals.d.ts index ba872ef46e7761..96790fb6652be5 100644 --- a/cli/tsc/dts/lib.deno.shared_globals.d.ts +++ b/cli/tsc/dts/lib.deno.shared_globals.d.ts @@ -15,14 +15,14 @@ /// /// -/** @category WASM */ +/** @category Wasm */ declare namespace WebAssembly { /** * The `WebAssembly.CompileError` object indicates an error during WebAssembly decoding or validation. * * [MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/CompileError) * - * @category WASM + * @category Wasm */ export class CompileError extends Error { /** Creates a new `WebAssembly.CompileError` object. */ @@ -36,7 +36,7 @@ declare namespace WebAssembly { * * [MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/Global) * - * @category WASM + * @category Wasm */ export class Global { /** Creates a new `Global` object. */ @@ -59,7 +59,7 @@ declare namespace WebAssembly { * * [MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/Instance) * - * @category WASM + * @category Wasm */ export class Instance { /** Creates a new Instance object. */ @@ -79,7 +79,7 @@ declare namespace WebAssembly { * * [MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/LinkError) * - * @category WASM + * @category Wasm */ export class LinkError extends Error { /** Creates a new WebAssembly.LinkError object. */ @@ -95,7 +95,7 @@ declare namespace WebAssembly { * * [MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/Memory) * - * @category WASM + * @category Wasm */ export class Memory { /** Creates a new `Memory` object. */ @@ -117,7 +117,7 @@ declare namespace WebAssembly { * * [MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/Module) * - * @category WASM + * @category Wasm */ export class Module { /** Creates a new `Module` object. */ @@ -145,7 +145,7 @@ declare namespace WebAssembly { * * [MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/RuntimeError) * - * @category WASM + * @category Wasm */ export class RuntimeError extends Error { /** Creates a new `WebAssembly.RuntimeError` object. */ @@ -160,7 +160,7 @@ declare namespace WebAssembly { * * [MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/Table) * - * @category WASM + * @category Wasm */ export class Table { /** Creates a new `Table` object. */ @@ -182,7 +182,7 @@ declare namespace WebAssembly { /** The `GlobalDescriptor` describes the options you can pass to * `new WebAssembly.Global()`. * - * @category WASM + * @category Wasm */ export interface GlobalDescriptor { mutable?: boolean; @@ -192,7 +192,7 @@ declare namespace WebAssembly { /** The `MemoryDescriptor` describes the options you can pass to * `new WebAssembly.Memory()`. * - * @category WASM + * @category Wasm */ export interface MemoryDescriptor { initial: number; @@ -203,7 +203,7 @@ declare namespace WebAssembly { /** A `ModuleExportDescriptor` is the description of a declared export in a * `WebAssembly.Module`. * - * @category WASM + * @category Wasm */ export interface ModuleExportDescriptor { kind: ImportExportKind; @@ -213,7 +213,7 @@ declare namespace WebAssembly { /** A `ModuleImportDescriptor` is the description of a declared import in a * `WebAssembly.Module`. * - * @category WASM + * @category Wasm */ export interface ModuleImportDescriptor { kind: ImportExportKind; @@ -224,7 +224,7 @@ declare namespace WebAssembly { /** The `TableDescriptor` describes the options you can pass to * `new WebAssembly.Table()`. * - * @category WASM + * @category Wasm */ export interface TableDescriptor { element: TableKind; @@ -234,7 +234,7 @@ declare namespace WebAssembly { /** The value returned from `WebAssembly.instantiate`. * - * @category WASM + * @category Wasm */ export interface WebAssemblyInstantiatedSource { /* A `WebAssembly.Instance` object that contains all the exported WebAssembly functions. */ @@ -247,21 +247,21 @@ declare namespace WebAssembly { module: Module; } - /** @category WASM */ + /** @category Wasm */ export type ImportExportKind = "function" | "global" | "memory" | "table"; - /** @category WASM */ + /** @category Wasm */ export type TableKind = "anyfunc"; - /** @category WASM */ + /** @category Wasm */ export type ValueType = "f32" | "f64" | "i32" | "i64"; - /** @category WASM */ + /** @category Wasm */ export type ExportValue = Function | Global | Memory | Table; - /** @category WASM */ + /** @category Wasm */ export type Exports = Record; - /** @category WASM */ + /** @category Wasm */ export type ImportValue = ExportValue | number; - /** @category WASM */ + /** @category Wasm */ export type ModuleImports = Record; - /** @category WASM */ + /** @category Wasm */ export type Imports = Record; /** @@ -272,7 +272,7 @@ declare namespace WebAssembly { * * [MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/compile) * - * @category WASM + * @category Wasm */ export function compile(bytes: BufferSource): Promise; @@ -284,7 +284,7 @@ declare namespace WebAssembly { * * [MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/compileStreaming) * - * @category WASM + * @category Wasm */ export function compileStreaming( source: Response | Promise, @@ -301,7 +301,7 @@ declare namespace WebAssembly { * * [MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/instantiate) * - * @category WASM + * @category Wasm */ export function instantiate( bytes: BufferSource, @@ -318,7 +318,7 @@ declare namespace WebAssembly { * * [MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/instantiate) * - * @category WASM + * @category Wasm */ export function instantiate( moduleObject: Module, @@ -332,7 +332,7 @@ declare namespace WebAssembly { * * [MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/instantiateStreaming) * - * @category WASM + * @category Wasm */ export function instantiateStreaming( response: Response | PromiseLike, @@ -346,7 +346,7 @@ declare namespace WebAssembly { * * [MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/validate) * - * @category WASM + * @category Wasm */ export function validate(bytes: BufferSource): boolean; } diff --git a/cli/tsc/dts/lib.deno.unstable.d.ts b/cli/tsc/dts/lib.deno.unstable.d.ts index 973a09d92ab378..6759856e6add49 100644 --- a/cli/tsc/dts/lib.deno.unstable.d.ts +++ b/cli/tsc/dts/lib.deno.unstable.d.ts @@ -1180,6 +1180,32 @@ declare namespace Deno { ...values: unknown[] ): Displayable; + /** + * Display a JPG or PNG image. + * + * ``` + * Deno.jupyter.image("./cat.jpg"); + * Deno.jupyter.image("./dog.png"); + * ``` + * + * @category Jupyter + * @experimental + */ + export function image(path: string): Displayable; + + /** + * Display a JPG or PNG image. + * + * ``` + * const img = Deno.readFileSync("./cat.jpg"); + * Deno.jupyter.image(img); + * ``` + * + * @category Jupyter + * @experimental + */ + export function image(data: Uint8Array): Displayable; + /** * Format an object for displaying in Deno * @@ -1225,6 +1251,73 @@ declare namespace Deno { export {}; // only export exports } + /** + * **UNSTABLE**: New API, yet to be vetted. + * + * APIs for working with the OpenTelemetry observability framework. Deno can + * export traces, metrics, and logs to OpenTelemetry compatible backends via + * the OTLP protocol. + * + * Deno automatically instruments the runtime with OpenTelemetry traces and + * metrics. This data is exported via OTLP to OpenTelemetry compatible + * backends. User logs from the `console` API are exported as OpenTelemetry + * logs via OTLP. + * + * User code can also create custom traces, metrics, and logs using the + * OpenTelemetry API. This is done using the official OpenTelemetry package + * for JavaScript: + * [`npm:@opentelemetry/api`](https://opentelemetry.io/docs/languages/js/). + * Deno integrates with this package to provide trace context propagation + * between native Deno APIs (like `Deno.serve` or `fetch`) and custom user + * code. Deno also provides APIs that allow exporting custom telemetry data + * via the same OTLP channel used by the Deno runtime. This is done using the + * [`jsr:@deno/otel`](https://jsr.io/@deno/otel) package. + * + * @example Using OpenTelemetry API to create custom traces + * ```ts,ignore + * import { trace } from "npm:@opentelemetry/api@1"; + * import "jsr:@deno/otel@0.0.2/register"; + * + * const tracer = trace.getTracer("example-tracer"); + * + * async function doWork() { + * return tracer.startActiveSpan("doWork", async (span) => { + * span.setAttribute("key", "value"); + * await new Promise((resolve) => setTimeout(resolve, 1000)); + * span.end(); + * }); + * } + * + * Deno.serve(async (req) => { + * await doWork(); + * const resp = await fetch("https://example.com"); + * return resp; + * }); + * ``` + * + * @category Telemetry + * @experimental + */ + export namespace telemetry { + /** + * A SpanExporter compatible with OpenTelemetry.js + * https://open-telemetry.github.io/opentelemetry-js/interfaces/_opentelemetry_sdk_trace_base.SpanExporter.html + * @category Telemetry + * @experimental + */ + export class SpanExporter {} + + /** + * A ContextManager compatible with OpenTelemetry.js + * https://open-telemetry.github.io/opentelemetry-js/interfaces/_opentelemetry_api.ContextManager.html + * @category Telemetry + * @experimental + */ + export class ContextManager {} + + export {}; // only export exports + } + export {}; // only export exports } diff --git a/cli/tsc/dts/lib.dom.d.ts b/cli/tsc/dts/lib.dom.d.ts index 0a2f9b9edaa6e0..2684735597a37a 100644 --- a/cli/tsc/dts/lib.dom.d.ts +++ b/cli/tsc/dts/lib.dom.d.ts @@ -18277,7 +18277,7 @@ declare var ReadableStream: { new(underlyingSource: UnderlyingByteSource, strategy?: { highWaterMark?: number }): ReadableStream; new(underlyingSource: UnderlyingDefaultSource, strategy?: QueuingStrategy): ReadableStream; new(underlyingSource?: UnderlyingSource, strategy?: QueuingStrategy): ReadableStream; - from(asyncIterable: AsyncIterable | Iterable>): ReadableStream; + from(asyncIterable: AsyncIterable | Iterable> & object): ReadableStream; }; /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBReader) */ diff --git a/cli/tsc/mod.rs b/cli/tsc/mod.rs index 0e3387494af4fa..a8e8d73b68b002 100644 --- a/cli/tsc/mod.rs +++ b/cli/tsc/mod.rs @@ -3,10 +3,13 @@ use crate::args::TsConfig; use crate::args::TypeCheckMode; use crate::cache::FastInsecureHasher; +use crate::cache::ModuleInfoCache; use crate::node; use crate::npm::CliNpmResolver; +use crate::resolver::CjsTracker; use crate::util::checksum; use crate::util::path::mapped_specifier_for_tsc; +use crate::worker::create_isolate_create_params; use deno_ast::MediaType; use deno_core::anyhow::anyhow; @@ -31,13 +34,15 @@ use deno_graph::GraphKind; use deno_graph::Module; use deno_graph::ModuleGraph; use deno_graph::ResolutionResolved; +use deno_resolver::npm::ResolvePkgFolderFromDenoReqError; +use deno_runtime::deno_fs; use deno_runtime::deno_node::NodeResolver; use deno_semver::npm::NpmPackageReqReference; use node_resolver::errors::NodeJsErrorCode; use node_resolver::errors::NodeJsErrorCoded; -use node_resolver::NodeModuleKind; -use node_resolver::NodeResolution; -use node_resolver::NodeResolutionMode; +use node_resolver::errors::PackageSubpathResolveError; +use node_resolver::NodeResolutionKind; +use node_resolver::ResolutionMode; use once_cell::sync::Lazy; use std::borrow::Cow; use std::collections::HashMap; @@ -45,6 +50,7 @@ use std::fmt; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; +use thiserror::Error; mod diagnostics; @@ -299,8 +305,81 @@ pub struct EmittedFile { pub media_type: MediaType, } +pub fn into_specifier_and_media_type( + specifier: Option, +) -> (ModuleSpecifier, MediaType) { + match specifier { + Some(specifier) => { + let media_type = MediaType::from_specifier(&specifier); + + (specifier, media_type) + } + None => ( + Url::parse("internal:///missing_dependency.d.ts").unwrap(), + MediaType::Dts, + ), + } +} + +#[derive(Debug)] +pub struct TypeCheckingCjsTracker { + cjs_tracker: Arc, + module_info_cache: Arc, +} + +impl TypeCheckingCjsTracker { + pub fn new( + cjs_tracker: Arc, + module_info_cache: Arc, + ) -> Self { + Self { + cjs_tracker, + module_info_cache, + } + } + + pub fn is_cjs( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + code: &Arc, + ) -> bool { + let maybe_is_script = self + .module_info_cache + .as_module_analyzer() + .analyze_sync(specifier, media_type, code) + .ok() + .map(|info| info.is_script); + maybe_is_script + .and_then(|is_script| { + self + .cjs_tracker + .is_cjs_with_known_is_script(specifier, media_type, is_script) + .ok() + }) + .unwrap_or_else(|| { + self + .cjs_tracker + .is_maybe_cjs(specifier, media_type) + .unwrap_or(false) + }) + } + + pub fn is_cjs_with_known_is_script( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + is_script: bool, + ) -> Result { + self + .cjs_tracker + .is_cjs_with_known_is_script(specifier, media_type, is_script) + } +} + #[derive(Debug)] pub struct RequestNpmState { + pub cjs_tracker: Arc, pub node_resolver: Arc, pub npm_resolver: Arc, } @@ -453,7 +532,7 @@ pub fn as_ts_script_kind(media_type: MediaType) -> i32 { MediaType::Tsx => 4, MediaType::Json => 6, MediaType::SourceMap - | MediaType::TsBuildInfo + | MediaType::Css | MediaType::Wasm | MediaType::Unknown => 0, } @@ -486,25 +565,22 @@ fn op_load_inner( ) -> Result, AnyError> { fn load_from_node_modules( specifier: &ModuleSpecifier, - node_resolver: Option<&NodeResolver>, + npm_state: Option<&RequestNpmState>, media_type: &mut MediaType, is_cjs: &mut bool, ) -> Result { *media_type = MediaType::from_specifier(specifier); - *is_cjs = node_resolver - .map(|node_resolver| { - match node_resolver.url_to_node_resolution(specifier.clone()) { - Ok(NodeResolution::CommonJs(_)) => true, - Ok(NodeResolution::Esm(_)) - | Ok(NodeResolution::BuiltIn(_)) - | Err(_) => false, - } - }) - .unwrap_or(false); let file_path = specifier.to_file_path().unwrap(); let code = std::fs::read_to_string(&file_path) .with_context(|| format!("Unable to load {}", file_path.display()))?; - Ok(code) + let code: Arc = code.into(); + *is_cjs = npm_state + .map(|npm_state| { + npm_state.cjs_tracker.is_cjs(specifier, *media_type, &code) + }) + .unwrap_or(false); + // todo(dsherret): how to avoid cloning here? + Ok(code.to_string()) } let state = state.borrow_mut::(); @@ -557,6 +633,13 @@ fn op_load_inner( match module { Module::Js(module) => { media_type = module.media_type; + if let Some(npm_state) = &state.maybe_npm { + is_cjs = npm_state.cjs_tracker.is_cjs_with_known_is_script( + specifier, + module.media_type, + module.is_script, + )?; + } let source = module .fast_check_module() .map(|m| &*m.source) @@ -567,17 +650,27 @@ fn op_load_inner( media_type = MediaType::Json; Some(Cow::Borrowed(&*module.source)) } + Module::Wasm(module) => { + media_type = MediaType::Dts; + Some(Cow::Borrowed(&*module.source_dts)) + } Module::Npm(_) | Module::Node(_) => None, Module::External(module) => { - // means it's Deno code importing an npm module - let specifier = - node::resolve_specifier_into_node_modules(&module.specifier); - Some(Cow::Owned(load_from_node_modules( - &specifier, - state.maybe_npm.as_ref().map(|n| n.node_resolver.as_ref()), - &mut media_type, - &mut is_cjs, - )?)) + if module.specifier.scheme() != "file" { + None + } else { + // means it's Deno code importing an npm module + let specifier = node::resolve_specifier_into_node_modules( + &module.specifier, + &deno_fs::RealFs, + ); + Some(Cow::Owned(load_from_node_modules( + &specifier, + state.maybe_npm.as_ref(), + &mut media_type, + &mut is_cjs, + )?)) + } } } } else if let Some(npm) = state @@ -587,7 +680,7 @@ fn op_load_inner( { Some(Cow::Owned(load_from_node_modules( specifier, - Some(npm.node_resolver.as_ref()), + Some(npm), &mut media_type, &mut is_cjs, )?)) @@ -614,10 +707,9 @@ pub struct ResolveArgs { /// The base specifier that the supplied specifier strings should be resolved /// relative to. pub base: String, - /// If the base is cjs. - pub is_base_cjs: bool, /// A list of specifiers that should be resolved. - pub specifiers: Vec, + /// (is_cjs: bool, raw_specifier: String) + pub specifiers: Vec<(bool, String)>, } #[op2] @@ -625,17 +717,9 @@ pub struct ResolveArgs { fn op_resolve( state: &mut OpState, #[string] base: String, - is_base_cjs: bool, - #[serde] specifiers: Vec, + #[serde] specifiers: Vec<(bool, String)>, ) -> Result, AnyError> { - op_resolve_inner( - state, - ResolveArgs { - base, - is_base_cjs, - specifiers, - }, - ) + op_resolve_inner(state, ResolveArgs { base, specifiers }) } #[inline] @@ -646,11 +730,6 @@ fn op_resolve_inner( let state = state.borrow_mut::(); let mut resolved: Vec<(String, &'static str)> = Vec::with_capacity(args.specifiers.len()); - let referrer_kind = if args.is_base_cjs { - NodeModuleKind::Cjs - } else { - NodeModuleKind::Esm - }; let referrer = if let Some(remapped_specifier) = state.remapped_specifiers.get(&args.base) { @@ -662,7 +741,8 @@ fn op_resolve_inner( "Error converting a string module specifier for \"op_resolve\".", )? }; - for specifier in args.specifiers { + let referrer_module = state.graph.get(&referrer); + for (is_cjs, specifier) in args.specifiers { if specifier.starts_with("node:") { resolved.push(( MISSING_DEPENDENCY_SPECIFIER.to_string(), @@ -677,23 +757,51 @@ fn op_resolve_inner( continue; } - let graph = &state.graph; - let resolved_dep = graph - .get(&referrer) + let resolved_dep = referrer_module .and_then(|m| m.js()) .and_then(|m| m.dependencies_prefer_fast_check().get(&specifier)) .and_then(|d| d.maybe_type.ok().or_else(|| d.maybe_code.ok())); + let resolution_mode = if is_cjs { + ResolutionMode::Require + } else { + ResolutionMode::Import + }; let maybe_result = match resolved_dep { Some(ResolutionResolved { specifier, .. }) => { - resolve_graph_specifier_types(specifier, &referrer, state)? + resolve_graph_specifier_types( + specifier, + &referrer, + // we could get this from the resolved dep, but for now assume + // the value resolved in TypeScript is better + resolution_mode, + state, + )? + } + _ => { + match resolve_non_graph_specifier_types( + &specifier, + &referrer, + resolution_mode, + state, + ) { + Ok(maybe_result) => maybe_result, + Err( + err @ ResolveNonGraphSpecifierTypesError::ResolvePkgFolderFromDenoReq( + ResolvePkgFolderFromDenoReqError::Managed(_), + ), + ) => { + // it's most likely requesting the jsxImportSource, which isn't loaded + // into the graph when not using jsx, so just ignore this error + if specifier.ends_with("/jsx-runtime") { + None + } else { + return Err(err.into()); + } + } + Err(err) => return Err(err.into()), + } } - _ => resolve_non_graph_specifier_types( - &specifier, - &referrer, - referrer_kind, - state, - )?, }; let result = match maybe_result { Some((specifier, media_type)) => { @@ -718,7 +826,13 @@ fn op_resolve_inner( } } }; - (specifier_str, media_type.as_ts_extension()) + ( + specifier_str, + match media_type { + MediaType::Css => ".js", // surface these as .js for typescript + media_type => media_type.as_ts_extension(), + }, + ) } None => ( MISSING_DEPENDENCY_SPECIFIER.to_string(), @@ -735,6 +849,7 @@ fn op_resolve_inner( fn resolve_graph_specifier_types( specifier: &ModuleSpecifier, referrer: &ModuleSpecifier, + resolution_mode: ResolutionMode, state: &State, ) -> Result, AnyError> { let graph = &state.graph; @@ -775,6 +890,9 @@ fn resolve_graph_specifier_types( Some(Module::Json(module)) => { Ok(Some((module.specifier.clone(), module.media_type))) } + Some(Module::Wasm(module)) => { + Ok(Some((module.specifier.clone(), MediaType::Dmts))) + } Some(Module::Npm(module)) => { if let Some(npm) = &state.maybe_npm.as_ref() { let package_folder = npm @@ -787,43 +905,53 @@ fn resolve_graph_specifier_types( &package_folder, module.nv_reference.sub_path(), Some(referrer), - NodeResolutionMode::Types, + resolution_mode, + NodeResolutionKind::Types, ); - let maybe_resolution = match res_result { - Ok(res) => Some(res), + let maybe_url = match res_result { + Ok(url) => Some(url), Err(err) => match err.code() { NodeJsErrorCode::ERR_TYPES_NOT_FOUND | NodeJsErrorCode::ERR_MODULE_NOT_FOUND => None, _ => return Err(err.into()), }, }; - Ok(Some(NodeResolution::into_specifier_and_media_type( - maybe_resolution, - ))) + Ok(Some(into_specifier_and_media_type(maybe_url))) } else { Ok(None) } } Some(Module::External(module)) => { // we currently only use "External" for when the module is in an npm package - Ok(state.maybe_npm.as_ref().map(|npm| { - let specifier = - node::resolve_specifier_into_node_modules(&module.specifier); - NodeResolution::into_specifier_and_media_type( - npm.node_resolver.url_to_node_resolution(specifier).ok(), - ) + Ok(state.maybe_npm.as_ref().map(|_| { + let specifier = node::resolve_specifier_into_node_modules( + &module.specifier, + &deno_fs::RealFs, + ); + into_specifier_and_media_type(Some(specifier)) })) } Some(Module::Node(_)) | None => Ok(None), } } +#[derive(Debug, Error)] +enum ResolveNonGraphSpecifierTypesError { + #[error(transparent)] + ResolvePkgFolderFromDenoReq(#[from] ResolvePkgFolderFromDenoReqError), + #[error(transparent)] + PackageSubpathResolve(#[from] PackageSubpathResolveError), +} + fn resolve_non_graph_specifier_types( raw_specifier: &str, referrer: &ModuleSpecifier, - referrer_kind: NodeModuleKind, + resolution_mode: ResolutionMode, state: &State, -) -> Result, AnyError> { +) -> Result< + Option<(ModuleSpecifier, MediaType)>, + ResolveNonGraphSpecifierTypesError, +> { let npm = match state.maybe_npm.as_ref() { Some(npm) => npm, None => return Ok(None), // we only support non-graph types for npm packages @@ -831,20 +959,21 @@ fn resolve_non_graph_specifier_types( let node_resolver = &npm.node_resolver; if node_resolver.in_npm_package(referrer) { // we're in an npm package, so use node resolution - Ok(Some(NodeResolution::into_specifier_and_media_type( + Ok(Some(into_specifier_and_media_type( node_resolver .resolve( raw_specifier, referrer, - referrer_kind, - NodeResolutionMode::Types, + resolution_mode, + NodeResolutionKind::Types, ) - .ok(), + .ok() + .map(|res| res.into_url()), ))) } else if let Ok(npm_req_ref) = NpmPackageReqReference::from_str(raw_specifier) { - debug_assert_eq!(referrer_kind, NodeModuleKind::Esm); + debug_assert_eq!(resolution_mode, ResolutionMode::Import); // todo(dsherret): add support for injecting this in the graph so // we don't need this special code here. // This could occur when resolving npm:@types/node when it is @@ -856,19 +985,18 @@ fn resolve_non_graph_specifier_types( &package_folder, npm_req_ref.sub_path(), Some(referrer), - NodeResolutionMode::Types, + resolution_mode, + NodeResolutionKind::Types, ); - let maybe_resolution = match res_result { - Ok(res) => Some(res), + let maybe_url = match res_result { + Ok(url) => Some(url), Err(err) => match err.code() { NodeJsErrorCode::ERR_TYPES_NOT_FOUND | NodeJsErrorCode::ERR_MODULE_NOT_FOUND => None, _ => return Err(err.into()), }, }; - Ok(Some(NodeResolution::into_specifier_and_media_type( - maybe_resolution, - ))) + Ok(Some(into_specifier_and_media_type(maybe_url))) } else { Ok(None) } @@ -981,6 +1109,7 @@ pub fn exec(request: Request) -> Result { root_map, remapped_specifiers, )], + create_params: create_isolate_create_params(), ..Default::default() }); @@ -1071,7 +1200,7 @@ mod tests { .context("Unable to get CWD") .unwrap(), ); - let mut op_state = OpState::new(None); + let mut op_state = OpState::new(None, None); op_state.put(state); op_state } @@ -1256,8 +1385,7 @@ mod tests { &mut state, ResolveArgs { base: "https://deno.land/x/a.ts".to_string(), - is_base_cjs: false, - specifiers: vec!["./b.ts".to_string()], + specifiers: vec![(false, "./b.ts".to_string())], }, ) .expect("should have invoked op"); @@ -1276,8 +1404,7 @@ mod tests { &mut state, ResolveArgs { base: "https://deno.land/x/a.ts".to_string(), - is_base_cjs: false, - specifiers: vec!["./bad.ts".to_string()], + specifiers: vec![(false, "./bad.ts".to_string())], }, ) .expect("should have not errored"); diff --git a/cli/util/extract.rs b/cli/util/extract.rs index 841cf6eb0f06aa..be68202aa1eecf 100644 --- a/cli/util/extract.rs +++ b/cli/util/extract.rs @@ -64,7 +64,7 @@ fn extract_inner( }) { Ok(parsed) => { let mut c = ExportCollector::default(); - c.visit_program(parsed.program_ref()); + c.visit_program(parsed.program().as_ref()); c } Err(_) => ExportCollector::default(), @@ -254,7 +254,11 @@ impl ExportCollector { let mut import_specifiers = vec![]; if let Some(default_export) = &self.default_export { - if !symbols_to_exclude.contains(default_export) { + // If the default export conflicts with a named export, a named one + // takes precedence. + if !symbols_to_exclude.contains(default_export) + && !self.named_exports.contains(default_export) + { import_specifiers.push(ast::ImportSpecifier::Default( ast::ImportDefaultSpecifier { span: DUMMY_SP, @@ -566,14 +570,14 @@ fn generate_pseudo_file( })?; let top_level_atoms = swc_utils::collect_decls_with_ctxt::( - parsed.program_ref(), + &parsed.program_ref(), parsed.top_level_context(), ); let transformed = parsed .program_ref() - .clone() + .to_owned() .fold_with(&mut as_folder(Transform { specifier: &file.specifier, base_file_specifier, @@ -582,7 +586,10 @@ fn generate_pseudo_file( wrap_kind, })); - let source = deno_ast::swc::codegen::to_code(&transformed); + let source = deno_ast::swc::codegen::to_code_with_comments( + Some(&parsed.comments().as_single_threaded()), + &transformed, + ); log::debug!("{}:\n{}", file.specifier, source); @@ -1137,6 +1144,57 @@ Deno.test("file:///README.md$6-12.js", async ()=>{ media_type: MediaType::JavaScript, }], }, + // https://github.com/denoland/deno/issues/26009 + Test { + input: Input { + source: r#" +/** + * ```ts + * console.log(Foo) + * ``` + */ +export class Foo {} +export default Foo +"#, + specifier: "file:///main.ts", + }, + expected: vec![Expected { + source: r#"import { Foo } from "file:///main.ts"; +Deno.test("file:///main.ts$3-6.ts", async ()=>{ + console.log(Foo); +}); +"#, + specifier: "file:///main.ts$3-6.ts", + media_type: MediaType::TypeScript, + }], + }, + // https://github.com/denoland/deno/issues/26728 + Test { + input: Input { + source: r#" +/** + * ```ts + * // @ts-expect-error: can only add numbers + * add('1', '2'); + * ``` + */ +export function add(first: number, second: number) { + return first + second; +} +"#, + specifier: "file:///main.ts", + }, + expected: vec![Expected { + source: r#"import { add } from "file:///main.ts"; +Deno.test("file:///main.ts$3-7.ts", async ()=>{ + // @ts-expect-error: can only add numbers + add('1', '2'); +}); +"#, + specifier: "file:///main.ts$3-7.ts", + media_type: MediaType::TypeScript, + }], + }, ]; for test in tests { @@ -1326,6 +1384,53 @@ assertEquals(add(1, 2), 3); media_type: MediaType::JavaScript, }], }, + // https://github.com/denoland/deno/issues/26009 + Test { + input: Input { + source: r#" +/** + * ```ts + * console.log(Foo) + * ``` + */ +export class Foo {} +export default Foo +"#, + specifier: "file:///main.ts", + }, + expected: vec![Expected { + source: r#"import { Foo } from "file:///main.ts"; +console.log(Foo); +"#, + specifier: "file:///main.ts$3-6.ts", + media_type: MediaType::TypeScript, + }], + }, + // https://github.com/denoland/deno/issues/26728 + Test { + input: Input { + source: r#" +/** + * ```ts + * // @ts-expect-error: can only add numbers + * add('1', '2'); + * ``` + */ +export function add(first: number, second: number) { + return first + second; +} +"#, + specifier: "file:///main.ts", + }, + expected: vec![Expected { + source: r#"import { add } from "file:///main.ts"; +// @ts-expect-error: can only add numbers +add('1', '2'); +"#, + specifier: "file:///main.ts$3-7.ts", + media_type: MediaType::TypeScript, + }], + }, ]; for test in tests { @@ -1366,7 +1471,7 @@ assertEquals(add(1, 2), 3); }) .unwrap(); - collector.visit_program(parsed.program_ref()); + parsed.program_ref().visit_with(&mut collector); collector } @@ -1581,6 +1686,16 @@ declare global { named_expected: atom_set!(), default_expected: None, }, + // The identifier `Foo` conflicts, but `ExportCollector` doesn't do + // anything about it. It is handled by `to_import_specifiers` method. + Test { + input: r#" +export class Foo {} +export default Foo +"#, + named_expected: atom_set!("Foo"), + default_expected: Some("Foo".into()), + }, ]; for test in tests { diff --git a/cli/util/file_watcher.rs b/cli/util/file_watcher.rs index d92d880bc1fa8b..b9318a6e4bd612 100644 --- a/cli/util/file_watcher.rs +++ b/cli/util/file_watcher.rs @@ -30,7 +30,7 @@ use tokio::sync::mpsc; use tokio::sync::mpsc::UnboundedReceiver; use tokio::time::sleep; -const CLEAR_SCREEN: &str = "\x1B[2J\x1B[1;1H"; +const CLEAR_SCREEN: &str = "\x1B[H\x1B[2J\x1B[3J"; const DEBOUNCE_INTERVAL: Duration = Duration::from_millis(200); struct DebouncedReceiver { @@ -73,7 +73,6 @@ impl DebouncedReceiver { } } -#[allow(clippy::print_stderr)] async fn error_handler(watch_future: F) -> bool where F: Future>, @@ -84,7 +83,7 @@ where Some(e) => format_js_error(e), None => format!("{err:?}"), }; - eprintln!( + log::error!( "{}: {}", colors::red_bold("error"), error_string.trim_start_matches("error: ") @@ -128,19 +127,12 @@ impl PrintConfig { } } -fn create_print_after_restart_fn( - banner: &'static str, - clear_screen: bool, -) -> impl Fn() { +fn create_print_after_restart_fn(clear_screen: bool) -> impl Fn() { move || { #[allow(clippy::print_stderr)] if clear_screen && std::io::stderr().is_terminal() { eprint!("{}", CLEAR_SCREEN); } - info!( - "{} File change detected! Restarting!", - colors::intense_blue(banner), - ); } } @@ -188,7 +180,17 @@ impl WatcherCommunicator { } pub fn print(&self, msg: String) { - log::info!("{} {}", self.banner, msg); + log::info!("{} {}", self.banner, colors::gray(msg)); + } + + pub fn show_path_changed(&self, changed_paths: Option>) { + if let Some(paths) = changed_paths { + if !paths.is_empty() { + self.print(format!("Restarting! File change detected: {:?}", paths[0])) + } else { + self.print("Restarting! File change detected.".to_string()) + } + } } } @@ -264,7 +266,7 @@ where clear_screen, } = print_config; - let print_after_restart = create_print_after_restart_fn(banner, clear_screen); + let print_after_restart = create_print_after_restart_fn(clear_screen); let watcher_communicator = Arc::new(WatcherCommunicator { paths_to_watch_tx: paths_to_watch_tx.clone(), changed_paths_rx: changed_paths_rx.resubscribe(), diff --git a/cli/util/fs.rs b/cli/util/fs.rs index fdf6035ecdec6a..ba84a0e8f33d06 100644 --- a/cli/util/fs.rs +++ b/cli/util/fs.rs @@ -20,7 +20,6 @@ use deno_core::error::AnyError; use deno_core::unsync::spawn_blocking; use deno_core::ModuleSpecifier; use deno_runtime::deno_fs::FileSystem; -use deno_runtime::deno_node::PathClean; use crate::util::path::get_atomic_file_path; use crate::util::progress_bar::ProgressBar; @@ -37,10 +36,98 @@ pub fn atomic_write_file_with_retries>( file_path: &Path, data: T, mode: u32, +) -> std::io::Result<()> { + struct RealAtomicWriteFileFs { + mode: u32, + } + + impl AtomicWriteFileFs for RealAtomicWriteFileFs { + fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()> { + write_file(path, bytes, self.mode) + } + fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()> { + std::fs::rename(from, to) + } + fn remove_file(&self, path: &Path) -> std::io::Result<()> { + std::fs::remove_file(path) + } + fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()> { + std::fs::create_dir_all(dir_path) + } + fn path_exists(&self, path: &Path) -> bool { + path.exists() + } + } + + atomic_write_file_with_retries_and_fs( + &RealAtomicWriteFileFs { mode }, + file_path, + data.as_ref(), + ) +} + +pub trait AtomicWriteFileFs { + fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()>; + fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()>; + fn remove_file(&self, path: &Path) -> std::io::Result<()>; + fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()>; + fn path_exists(&self, path: &Path) -> bool; +} + +pub struct AtomicWriteFileFsAdapter<'a> { + pub fs: &'a dyn FileSystem, + pub write_mode: u32, +} + +impl<'a> AtomicWriteFileFs for AtomicWriteFileFsAdapter<'a> { + fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()> { + self + .fs + .write_file_sync( + path, + deno_runtime::deno_fs::OpenOptions::write( + true, + false, + false, + Some(self.write_mode), + ), + None, + bytes, + ) + .map_err(|e| e.into_io_error()) + } + + fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()> { + self.fs.rename_sync(from, to).map_err(|e| e.into_io_error()) + } + + fn remove_file(&self, path: &Path) -> std::io::Result<()> { + self + .fs + .remove_sync(path, false) + .map_err(|e| e.into_io_error()) + } + + fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()> { + self + .fs + .mkdir_sync(dir_path, /* recursive */ true, None) + .map_err(|e| e.into_io_error()) + } + + fn path_exists(&self, path: &Path) -> bool { + self.fs.exists_sync(path) + } +} + +pub fn atomic_write_file_with_retries_and_fs>( + fs: &impl AtomicWriteFileFs, + file_path: &Path, + data: T, ) -> std::io::Result<()> { let mut count = 0; loop { - match atomic_write_file(file_path, data.as_ref(), mode) { + match atomic_write_file(fs, file_path, data.as_ref()) { Ok(()) => return Ok(()), Err(err) => { if count >= 5 { @@ -61,63 +148,54 @@ pub fn atomic_write_file_with_retries>( /// /// This also handles creating the directory if a NotFound error /// occurs. -fn atomic_write_file>( +fn atomic_write_file( + fs: &impl AtomicWriteFileFs, file_path: &Path, - data: T, - mode: u32, + data: &[u8], ) -> std::io::Result<()> { fn atomic_write_file_raw( + fs: &impl AtomicWriteFileFs, temp_file_path: &Path, file_path: &Path, data: &[u8], - mode: u32, ) -> std::io::Result<()> { - write_file(temp_file_path, data, mode)?; - std::fs::rename(temp_file_path, file_path).map_err(|err| { - // clean up the created temp file on error - let _ = std::fs::remove_file(temp_file_path); - err - }) + fs.write_file(temp_file_path, data)?; + fs.rename_file(temp_file_path, file_path) + .inspect_err(|_err| { + // clean up the created temp file on error + let _ = fs.remove_file(temp_file_path); + }) } - fn inner(file_path: &Path, data: &[u8], mode: u32) -> std::io::Result<()> { - let temp_file_path = get_atomic_file_path(file_path); + let temp_file_path = get_atomic_file_path(file_path); - if let Err(write_err) = - atomic_write_file_raw(&temp_file_path, file_path, data, mode) - { - if write_err.kind() == ErrorKind::NotFound { - let parent_dir_path = file_path.parent().unwrap(); - match std::fs::create_dir_all(parent_dir_path) { - Ok(()) => { - return atomic_write_file_raw( - &temp_file_path, - file_path, - data, - mode, - ) + if let Err(write_err) = + atomic_write_file_raw(fs, &temp_file_path, file_path, data) + { + if write_err.kind() == ErrorKind::NotFound { + let parent_dir_path = file_path.parent().unwrap(); + match fs.create_dir_all(parent_dir_path) { + Ok(()) => { + return atomic_write_file_raw(fs, &temp_file_path, file_path, data) .map_err(|err| add_file_context_to_err(file_path, err)); - } - Err(create_err) => { - if !parent_dir_path.exists() { - return Err(Error::new( - create_err.kind(), - format!( - "{:#} (for '{}')\nCheck the permission of the directory.", - create_err, - parent_dir_path.display() - ), - )); - } + } + Err(create_err) => { + if !fs.path_exists(parent_dir_path) { + return Err(Error::new( + create_err.kind(), + format!( + "{:#} (for '{}')\nCheck the permission of the directory.", + create_err, + parent_dir_path.display() + ), + )); } } } - return Err(add_file_context_to_err(file_path, write_err)); } - Ok(()) + return Err(add_file_context_to_err(file_path, write_err)); } - - inner(file_path, data.as_ref(), mode) + Ok(()) } /// Creates a std::fs::File handling if the parent does not exist. @@ -199,7 +277,7 @@ pub fn write_file_2>( /// Similar to `std::fs::canonicalize()` but strips UNC prefixes on Windows. pub fn canonicalize_path(path: &Path) -> Result { - Ok(deno_core::strip_unc_prefix(path.canonicalize()?)) + Ok(deno_path_util::strip_unc_prefix(path.canonicalize()?)) } /// Canonicalizes a path which might be non-existent by going up the @@ -211,48 +289,18 @@ pub fn canonicalize_path(path: &Path) -> Result { pub fn canonicalize_path_maybe_not_exists( path: &Path, ) -> Result { - canonicalize_path_maybe_not_exists_with_custom_fn(path, canonicalize_path) + deno_path_util::canonicalize_path_maybe_not_exists(path, &canonicalize_path) } pub fn canonicalize_path_maybe_not_exists_with_fs( path: &Path, fs: &dyn FileSystem, ) -> Result { - canonicalize_path_maybe_not_exists_with_custom_fn(path, |path| { + deno_path_util::canonicalize_path_maybe_not_exists(path, &|path| { fs.realpath_sync(path).map_err(|err| err.into_io_error()) }) } -fn canonicalize_path_maybe_not_exists_with_custom_fn( - path: &Path, - canonicalize: impl Fn(&Path) -> Result, -) -> Result { - let path = path.to_path_buf().clean(); - let mut path = path.as_path(); - let mut names_stack = Vec::new(); - loop { - match canonicalize(path) { - Ok(mut canonicalized_path) => { - for name in names_stack.into_iter().rev() { - canonicalized_path = canonicalized_path.join(name); - } - return Ok(canonicalized_path); - } - Err(err) if err.kind() == ErrorKind::NotFound => { - names_stack.push(match path.file_name() { - Some(name) => name.to_owned(), - None => return Err(err), - }); - path = match path.parent() { - Some(parent) => parent, - None => return Err(err), - }; - } - Err(err) => return Err(err), - } - } -} - /// Collects module specifiers that satisfy the given predicate as a file path, by recursively walking `include`. /// Specifiers that start with http and https are left intact. /// Note: This ignores all .git and node_modules folders. @@ -517,7 +565,9 @@ pub fn symlink_dir(oldpath: &Path, newpath: &Path) -> Result<(), Error> { use std::os::windows::fs::symlink_dir; symlink_dir(oldpath, newpath).map_err(|err| { if let Some(code) = err.raw_os_error() { - if code as u32 == winapi::shared::winerror::ERROR_PRIVILEGE_NOT_HELD { + if code as u32 == winapi::shared::winerror::ERROR_PRIVILEGE_NOT_HELD + || code as u32 == winapi::shared::winerror::ERROR_INVALID_FUNCTION + { return err_mapper(err, Some(ErrorKind::PermissionDenied)); } } @@ -609,7 +659,7 @@ impl LaxSingleProcessFsFlag { // // This uses a blocking task because we use a single threaded // runtime and this is time sensitive so we don't want it to update - // at the whims of of whatever is occurring on the runtime thread. + // at the whims of whatever is occurring on the runtime thread. spawn_blocking({ let token = token.clone(); let last_updated_path = last_updated_path.clone(); @@ -708,8 +758,8 @@ pub fn specifier_from_file_path( mod tests { use super::*; use deno_core::futures; - use deno_core::normalize_path; use deno_core::parking_lot::Mutex; + use deno_path_util::normalize_path; use pretty_assertions::assert_eq; use test_util::PathRef; use test_util::TempDir; diff --git a/cli/util/logger.rs b/cli/util/logger.rs index cdc89411fe1b77..2b8987c3e7e2cc 100644 --- a/cli/util/logger.rs +++ b/cli/util/logger.rs @@ -29,6 +29,7 @@ impl log::Log for CliLogger { // thread's state DrawThread::hide(); self.0.log(record); + deno_telemetry::handle_log(record); DrawThread::show(); } } @@ -65,6 +66,8 @@ pub fn init(maybe_level: Option) { .filter_module("swc_ecma_parser", log::LevelFilter::Error) // Suppress span lifecycle logs since they are too verbose .filter_module("tracing::span", log::LevelFilter::Off) + // for deno_compile, this is too verbose + .filter_module("editpe", log::LevelFilter::Error) .format(|buf, record| { let mut target = record.target().to_string(); if let Some(line_no) = record.line() { diff --git a/cli/util/mod.rs b/cli/util/mod.rs index e59b09d2c7167e..f81a74c449d026 100644 --- a/cli/util/mod.rs +++ b/cli/util/mod.rs @@ -14,6 +14,7 @@ pub mod logger; pub mod path; pub mod progress_bar; pub mod result; +pub mod retry; pub mod sync; pub mod text_encoding; pub mod unix; diff --git a/cli/util/path.rs b/cli/util/path.rs index 6f09cf1eacb9f7..173f357c087f42 100644 --- a/cli/util/path.rs +++ b/cli/util/path.rs @@ -27,7 +27,16 @@ pub fn is_importable_ext(path: &Path) -> bool { if let Some(ext) = get_extension(path) { matches!( ext.as_str(), - "ts" | "tsx" | "js" | "jsx" | "mjs" | "mts" | "cjs" | "cts" | "json" + "ts" + | "tsx" + | "js" + | "jsx" + | "mjs" + | "mts" + | "cjs" + | "cts" + | "json" + | "wasm" ) } else { false @@ -42,21 +51,6 @@ pub fn get_extension(file_path: &Path) -> Option { .map(|e| e.to_lowercase()); } -pub fn specifier_has_extension( - specifier: &ModuleSpecifier, - searching_ext: &str, -) -> bool { - let Some((_, ext)) = specifier.path().rsplit_once('.') else { - return false; - }; - let searching_ext = searching_ext.strip_prefix('.').unwrap_or(searching_ext); - debug_assert!(!searching_ext.contains('.')); // exts like .d.ts are not implemented here - if ext.len() != searching_ext.len() { - return false; - } - ext.eq_ignore_ascii_case(searching_ext) -} - pub fn get_atomic_dir_path(file_path: &Path) -> PathBuf { let rand = gen_rand_path_component(); let new_file_name = format!( @@ -165,48 +159,6 @@ pub fn relative_path(from: &Path, to: &Path) -> Option { pathdiff::diff_paths(to, from) } -/// Gets if the provided character is not supported on all -/// kinds of file systems. -pub fn is_banned_path_char(c: char) -> bool { - matches!(c, '<' | '>' | ':' | '"' | '|' | '?' | '*') -} - -/// Gets a safe local directory name for the provided url. -/// -/// For example: -/// https://deno.land:8080/path -> deno.land_8080/path -pub fn root_url_to_safe_local_dirname(root: &ModuleSpecifier) -> PathBuf { - fn sanitize_segment(text: &str) -> String { - text - .chars() - .map(|c| if is_banned_segment_char(c) { '_' } else { c }) - .collect() - } - - fn is_banned_segment_char(c: char) -> bool { - matches!(c, '/' | '\\') || is_banned_path_char(c) - } - - let mut result = String::new(); - if let Some(domain) = root.domain() { - result.push_str(&sanitize_segment(domain)); - } - if let Some(port) = root.port() { - if !result.is_empty() { - result.push('_'); - } - result.push_str(&port.to_string()); - } - let mut result = PathBuf::from(result); - if let Some(segments) = root.path_segments() { - for segment in segments.filter(|s| !s.is_empty()) { - result = result.join(sanitize_segment(segment)); - } - } - - result -} - /// Slightly different behaviour than the default matching /// where an exact path needs to be matched to be opted-in /// rather than just a partial directory match. @@ -279,6 +231,7 @@ mod test { assert!(is_script_ext(Path::new("foo.cjs"))); assert!(is_script_ext(Path::new("foo.cts"))); assert!(!is_script_ext(Path::new("foo.json"))); + assert!(!is_script_ext(Path::new("foo.wasm"))); assert!(!is_script_ext(Path::new("foo.mjsx"))); } @@ -300,6 +253,7 @@ mod test { assert!(is_importable_ext(Path::new("foo.cjs"))); assert!(is_importable_ext(Path::new("foo.cts"))); assert!(is_importable_ext(Path::new("foo.json"))); + assert!(is_importable_ext(Path::new("foo.wasm"))); assert!(!is_importable_ext(Path::new("foo.mjsx"))); } @@ -392,18 +346,6 @@ mod test { } } - #[test] - fn test_specifier_has_extension() { - fn get(specifier: &str, ext: &str) -> bool { - specifier_has_extension(&ModuleSpecifier::parse(specifier).unwrap(), ext) - } - - assert!(get("file:///a/b/c.ts", "ts")); - assert!(get("file:///a/b/c.ts", ".ts")); - assert!(!get("file:///a/b/c.ts", ".cts")); - assert!(get("file:///a/b/c.CtS", ".cts")); - } - #[test] fn test_to_percent_decoded_str() { let str = to_percent_decoded_str("%F0%9F%A6%95"); diff --git a/cli/util/progress_bar/renderer.rs b/cli/util/progress_bar/renderer.rs index a83ceb33340517..6b08dada12949c 100644 --- a/cli/util/progress_bar/renderer.rs +++ b/cli/util/progress_bar/renderer.rs @@ -193,10 +193,16 @@ impl ProgressBarRenderer for TextOnlyProgressBarRenderer { } }; + // TODO(@marvinhagemeister): We're trying to reconstruct the original + // specifier from the resolved one, but we lack the information about + // private registries URLs and other things here. let message = display_entry .message .replace("https://registry.npmjs.org/", "npm:") - .replace("https://jsr.io/", "jsr:"); + .replace("https://jsr.io/", "jsr:") + .replace("%2f", "/") + .replace("%2F", "/"); + display_str.push_str( &colors::gray(format!(" - {}{}\n", message, bytes_text)).to_string(), ); diff --git a/cli/util/retry.rs b/cli/util/retry.rs new file mode 100644 index 00000000000000..a8febe60dedf33 --- /dev/null +++ b/cli/util/retry.rs @@ -0,0 +1,41 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::future::Future; +use std::time::Duration; + +pub fn retry< + F: FnMut() -> Fut, + T, + E, + Fut: Future>, + ShouldRetry: FnMut(&E) -> bool, +>( + mut f: F, + mut should_retry: ShouldRetry, +) -> impl Future> { + const WAITS: [Duration; 3] = [ + Duration::from_millis(100), + Duration::from_millis(250), + Duration::from_millis(500), + ]; + + let mut waits = WAITS.into_iter(); + async move { + let mut first_result = None; + loop { + let result = f().await; + match result { + Ok(r) => return Ok(r), + Err(e) if !should_retry(&e) => return Err(e), + _ => {} + } + if first_result.is_none() { + first_result = Some(result); + } + let Some(wait) = waits.next() else { + return first_result.unwrap(); + }; + tokio::time::sleep(wait).await; + } + } +} diff --git a/cli/util/text_encoding.rs b/cli/util/text_encoding.rs index d2e0832c950a53..06b311e1501838 100644 --- a/cli/util/text_encoding.rs +++ b/cli/util/text_encoding.rs @@ -1,6 +1,8 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::borrow::Cow; use std::ops::Range; +use std::sync::Arc; use base64::prelude::BASE64_STANDARD; use base64::Engine; @@ -9,6 +11,24 @@ use deno_core::ModuleSourceCode; static SOURCE_MAP_PREFIX: &[u8] = b"//# sourceMappingURL=data:application/json;base64,"; +#[inline(always)] +pub fn from_utf8_lossy_cow(bytes: Cow<[u8]>) -> Cow { + match bytes { + Cow::Borrowed(bytes) => String::from_utf8_lossy(bytes), + Cow::Owned(bytes) => Cow::Owned(from_utf8_lossy_owned(bytes)), + } +} + +#[inline(always)] +pub fn from_utf8_lossy_owned(bytes: Vec) -> String { + match String::from_utf8_lossy(&bytes) { + Cow::Owned(code) => code, + // SAFETY: `String::from_utf8_lossy` guarantees that the result is valid + // UTF-8 if `Cow::Borrowed` is returned. + Cow::Borrowed(_) => unsafe { String::from_utf8_unchecked(bytes) }, + } +} + pub fn source_map_from_code(code: &[u8]) -> Option> { let range = find_source_map_range(code)?; let source_map_range = &code[range]; @@ -85,6 +105,29 @@ fn find_source_map_range(code: &[u8]) -> Option> { } } +/// Converts an `Arc` to an `Arc<[u8]>`. +#[allow(dead_code)] +pub fn arc_str_to_bytes(arc_str: Arc) -> Arc<[u8]> { + let raw = Arc::into_raw(arc_str); + // SAFETY: This is safe because they have the same memory layout. + unsafe { Arc::from_raw(raw as *const [u8]) } +} + +/// Converts an `Arc` to an `Arc` if able. +#[allow(dead_code)] +pub fn arc_u8_to_arc_str( + arc_u8: Arc<[u8]>, +) -> Result, std::str::Utf8Error> { + // Check that the string is valid UTF-8. + std::str::from_utf8(&arc_u8)?; + // SAFETY: the string is valid UTF-8, and the layout Arc<[u8]> is the same as + // Arc. This is proven by the From> impl for Arc<[u8]> from the + // standard library. + Ok(unsafe { + std::mem::transmute::, std::sync::Arc>(arc_u8) + }) +} + #[cfg(test)] mod tests { use std::sync::Arc; diff --git a/cli/util/v8.rs b/cli/util/v8.rs index fb16e67b77c203..6e690e6f30d9a3 100644 --- a/cli/util/v8.rs +++ b/cli/util/v8.rs @@ -46,15 +46,14 @@ pub fn init_v8_flags( .skip(1) .collect::>(); - #[allow(clippy::print_stderr)] if !unrecognized_v8_flags.is_empty() { for f in unrecognized_v8_flags { - eprintln!("error: V8 did not recognize flag '{f}'"); + log::error!("error: V8 did not recognize flag '{f}'"); } - eprintln!("\nFor a list of V8 flags, use '--v8-flags=--help'"); - std::process::exit(1); + log::error!("\nFor a list of V8 flags, use '--v8-flags=--help'"); + deno_runtime::exit(1); } if v8_flags_includes_help { - std::process::exit(0); + deno_runtime::exit(0); } } diff --git a/cli/worker.rs b/cli/worker.rs index 6176398d54778f..161d8bcc211e4d 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -14,34 +14,38 @@ use deno_core::v8; use deno_core::CompiledWasmModuleStore; use deno_core::Extension; use deno_core::FeatureChecker; -use deno_core::ModuleId; use deno_core::ModuleLoader; use deno_core::PollEventLoopOptions; use deno_core::SharedArrayBufferStore; use deno_runtime::code_cache; use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel; use deno_runtime::deno_fs; -use deno_runtime::deno_node; use deno_runtime::deno_node::NodeExtInitServices; +use deno_runtime::deno_node::NodeRequireLoader; +use deno_runtime::deno_node::NodeRequireLoaderRc; use deno_runtime::deno_node::NodeResolver; +use deno_runtime::deno_node::PackageJsonResolver; use deno_runtime::deno_permissions::PermissionsContainer; use deno_runtime::deno_tls::RootCertStoreProvider; use deno_runtime::deno_web::BlobStore; use deno_runtime::fmt_errors::format_js_error; use deno_runtime::inspector_server::InspectorServer; +use deno_runtime::ops::process::NpmProcessStateProviderRc; use deno_runtime::ops::worker_host::CreateWebWorkerCb; -use deno_runtime::permissions::RuntimePermissionDescriptorParser; use deno_runtime::web_worker::WebWorker; use deno_runtime::web_worker::WebWorkerOptions; +use deno_runtime::web_worker::WebWorkerServiceOptions; use deno_runtime::worker::MainWorker; use deno_runtime::worker::WorkerOptions; +use deno_runtime::worker::WorkerServiceOptions; use deno_runtime::BootstrapOptions; use deno_runtime::WorkerExecutionMode; use deno_runtime::WorkerLogLevel; use deno_semver::npm::NpmPackageReqReference; +use deno_telemetry::OtelConfig; use deno_terminal::colors; -use node_resolver::NodeResolution; -use node_resolver::NodeResolutionMode; +use node_resolver::NodeResolutionKind; +use node_resolver::ResolutionMode; use tokio::select; use crate::args::CliLockfile; @@ -54,22 +58,22 @@ use crate::util::file_watcher::WatcherCommunicator; use crate::util::file_watcher::WatcherRestartMode; use crate::version; -pub struct ModuleLoaderAndSourceMapGetter { +pub struct CreateModuleLoaderResult { pub module_loader: Rc, + pub node_require_loader: Rc, } pub trait ModuleLoaderFactory: Send + Sync { fn create_for_main( &self, root_permissions: PermissionsContainer, - dynamic_permissions: PermissionsContainer, - ) -> ModuleLoaderAndSourceMapGetter; + ) -> CreateModuleLoaderResult; fn create_for_worker( &self, - root_permissions: PermissionsContainer, - dynamic_permissions: PermissionsContainer, - ) -> ModuleLoaderAndSourceMapGetter; + parent_permissions: PermissionsContainer, + permissions: PermissionsContainer, + ) -> CreateModuleLoaderResult; } #[async_trait::async_trait(?Send)] @@ -79,6 +83,15 @@ pub trait HmrRunner: Send + Sync { async fn run(&mut self) -> Result<(), AnyError>; } +pub trait CliCodeCache: code_cache::CodeCache { + /// Gets if the code cache is still enabled. + fn enabled(&self) -> bool { + true + } + + fn as_code_cache(self: Arc) -> Arc; +} + #[async_trait::async_trait(?Send)] pub trait CoverageCollector: Send + Sync { async fn start_collecting(&mut self) -> Result<(), AnyError>; @@ -106,7 +119,6 @@ pub struct CliMainWorkerOptions { pub inspect_wait: bool, pub strace_ops: Option>, pub is_inspecting: bool, - pub is_npm_main: bool, pub location: Option, pub argv0: Option, pub node_debug: Option, @@ -124,7 +136,7 @@ pub struct CliMainWorkerOptions { struct SharedWorkerState { blob_store: Arc, broadcast_channel: InMemoryBroadcastChannel, - code_cache: Option>, + code_cache: Option>, compiled_wasm_module_store: CompiledWasmModuleStore, feature_checker: Arc, fs: Arc, @@ -134,31 +146,36 @@ struct SharedWorkerState { module_loader_factory: Box, node_resolver: Arc, npm_resolver: Arc, - permission_desc_parser: Arc, + pkg_json_resolver: Arc, root_cert_store_provider: Arc, + root_permissions: PermissionsContainer, shared_array_buffer_store: SharedArrayBufferStore, storage_key_resolver: StorageKeyResolver, options: CliMainWorkerOptions, subcommand: DenoSubcommand, + otel_config: Option, // `None` means OpenTelemetry is disabled. } impl SharedWorkerState { - pub fn create_node_init_services(&self) -> NodeExtInitServices { + pub fn create_node_init_services( + &self, + node_require_loader: NodeRequireLoaderRc, + ) -> NodeExtInitServices { NodeExtInitServices { - node_require_resolver: self.npm_resolver.clone().into_require_resolver(), + node_require_loader, node_resolver: self.node_resolver.clone(), - npm_process_state_provider: self - .npm_resolver - .clone() - .into_process_state_provider(), - npm_resolver: self.npm_resolver.clone().into_npm_resolver(), + npm_resolver: self.npm_resolver.clone().into_npm_pkg_folder_resolver(), + pkg_json_resolver: self.pkg_json_resolver.clone(), } } + + pub fn npm_process_state_provider(&self) -> NpmProcessStateProviderRc { + self.npm_resolver.clone().into_process_state_provider() + } } pub struct CliMainWorker { main_module: ModuleSpecifier, - is_main_cjs: bool, worker: MainWorker, shared: Arc, } @@ -180,17 +197,7 @@ impl CliMainWorker { log::debug!("main_module {}", self.main_module); - if self.is_main_cjs { - deno_node::load_cjs_module( - &mut self.worker.js_runtime, - &self.main_module.to_file_path().unwrap().to_string_lossy(), - true, - self.shared.options.inspect_brk, - )?; - } else { - self.execute_main_module_possibly_with_npm().await?; - } - + self.execute_main_module().await?; self.worker.dispatch_load_event()?; loop { @@ -278,22 +285,7 @@ impl CliMainWorker { /// Execute the given main module emitting load and unload events before and after execution /// respectively. pub async fn execute(&mut self) -> Result<(), AnyError> { - if self.inner.is_main_cjs { - deno_node::load_cjs_module( - &mut self.inner.worker.js_runtime, - &self - .inner - .main_module - .to_file_path() - .unwrap() - .to_string_lossy(), - true, - self.inner.shared.options.inspect_brk, - )?; - } else { - self.inner.execute_main_module_possibly_with_npm().await?; - } - + self.inner.execute_main_module().await?; self.inner.worker.dispatch_load_event()?; self.pending_unload = true; @@ -334,24 +326,13 @@ impl CliMainWorker { executor.execute().await } - pub async fn execute_main_module_possibly_with_npm( - &mut self, - ) -> Result<(), AnyError> { + pub async fn execute_main_module(&mut self) -> Result<(), AnyError> { let id = self.worker.preload_main_module(&self.main_module).await?; - self.evaluate_module_possibly_with_npm(id).await + self.worker.evaluate_module(id).await } - pub async fn execute_side_module_possibly_with_npm( - &mut self, - ) -> Result<(), AnyError> { + pub async fn execute_side_module(&mut self) -> Result<(), AnyError> { let id = self.worker.preload_side_module(&self.main_module).await?; - self.evaluate_module_possibly_with_npm(id).await - } - - async fn evaluate_module_possibly_with_npm( - &mut self, - id: ModuleId, - ) -> Result<(), AnyError> { self.worker.evaluate_module(id).await } @@ -412,6 +393,13 @@ impl CliMainWorker { } } +// TODO(bartlomieju): this should be moved to some other place, added to avoid string +// duplication between worker setups and `deno info` output. +pub fn get_cache_storage_dir() -> PathBuf { + // Note: we currently use temp_dir() to avoid managing storage size. + std::env::temp_dir().join("deno_cache") +} + #[derive(Clone)] pub struct CliMainWorkerFactory { shared: Arc, @@ -421,7 +409,7 @@ impl CliMainWorkerFactory { #[allow(clippy::too_many_arguments)] pub fn new( blob_store: Arc, - code_cache: Option>, + code_cache: Option>, feature_checker: Arc, fs: Arc, maybe_file_watcher_communicator: Option>, @@ -430,11 +418,13 @@ impl CliMainWorkerFactory { module_loader_factory: Box, node_resolver: Arc, npm_resolver: Arc, - permission_parser: Arc, + pkg_json_resolver: Arc, root_cert_store_provider: Arc, + root_permissions: PermissionsContainer, storage_key_resolver: StorageKeyResolver, subcommand: DenoSubcommand, options: CliMainWorkerOptions, + otel_config: Option, ) -> Self { Self { shared: Arc::new(SharedWorkerState { @@ -450,12 +440,14 @@ impl CliMainWorkerFactory { module_loader_factory, node_resolver, npm_resolver, - permission_desc_parser: permission_parser, + pkg_json_resolver, root_cert_store_provider, + root_permissions, shared_array_buffer_store: Default::default(), storage_key_resolver, options, subcommand, + otel_config, }), } } @@ -464,13 +456,12 @@ impl CliMainWorkerFactory { &self, mode: WorkerExecutionMode, main_module: ModuleSpecifier, - permissions: PermissionsContainer, ) -> Result { self .create_custom_worker( mode, main_module, - permissions, + self.shared.root_permissions.clone(), vec![], Default::default(), ) @@ -486,7 +477,13 @@ impl CliMainWorkerFactory { stdio: deno_runtime::deno_io::Stdio, ) -> Result { let shared = &self.shared; - let (main_module, is_main_cjs) = if let Ok(package_ref) = + let CreateModuleLoaderResult { + module_loader, + node_require_loader, + } = shared + .module_loader_factory + .create_for_main(permissions.clone()); + let main_module = if let Ok(package_ref) = NpmPackageReqReference::from_specifier(&main_module) { if let Some(npm_resolver) = shared.npm_resolver.as_managed() { @@ -506,9 +503,8 @@ impl CliMainWorkerFactory { package_ref.req(), &referrer, )?; - let node_resolution = self + let main_module = self .resolve_binary_entrypoint(&package_folder, package_ref.sub_path())?; - let is_main_cjs = matches!(node_resolution, NodeResolution::CommonJs(_)); if let Some(lockfile) = &shared.maybe_lockfile { // For npm binary commands, ensure that the lockfile gets updated @@ -517,26 +513,11 @@ impl CliMainWorkerFactory { lockfile.write_if_changed()?; } - (node_resolution.into_url(), is_main_cjs) - } else if shared.options.is_npm_main - || shared.node_resolver.in_npm_package(&main_module) - { - let node_resolution = - shared.node_resolver.url_to_node_resolution(main_module)?; - let is_main_cjs = matches!(node_resolution, NodeResolution::CommonJs(_)); - (node_resolution.into_url(), is_main_cjs) + main_module } else { - let is_cjs = main_module.path().ends_with(".cjs"); - (main_module, is_cjs) + main_module }; - let ModuleLoaderAndSourceMapGetter { module_loader } = - shared.module_loader_factory.create_for_main( - PermissionsContainer::allow_all( - self.shared.permission_desc_parser.clone(), - ), - permissions.clone(), - ); let maybe_inspector_server = shared.maybe_inspector_server.clone(); let create_web_worker_cb = @@ -555,10 +536,7 @@ impl CliMainWorkerFactory { }); let cache_storage_dir = maybe_storage_key.map(|key| { // TODO(@satyarohith): storage quota management - // Note: we currently use temp_dir() to avoid managing storage size. - std::env::temp_dir() - .join("deno_cache") - .join(checksum::gen(&[key.as_bytes()])) + get_cache_storage_dir().join(checksum::gen(&[key.as_bytes()])) }); // TODO(bartlomieju): this is cruft, update FeatureChecker to spit out @@ -572,6 +550,26 @@ impl CliMainWorkerFactory { } } + let services = WorkerServiceOptions { + root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), + module_loader, + fs: shared.fs.clone(), + node_services: Some( + shared.create_node_init_services(node_require_loader), + ), + npm_process_state_provider: Some(shared.npm_process_state_provider()), + blob_store: shared.blob_store.clone(), + broadcast_channel: shared.broadcast_channel.clone(), + fetch_dns_resolver: Default::default(), + shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()), + compiled_wasm_module_store: Some( + shared.compiled_wasm_module_store.clone(), + ), + feature_checker, + permissions, + v8_code_cache: shared.code_cache.clone().map(|c| c.as_code_cache()), + }; + let options = WorkerOptions { bootstrap: BootstrapOptions { deno_version: crate::version::DENO_VERSION_INFO.deno.to_string(), @@ -598,15 +596,15 @@ impl CliMainWorkerFactory { mode, serve_port: shared.options.serve_port, serve_host: shared.options.serve_host.clone(), + otel_config: shared.otel_config.clone(), }, extensions: custom_extensions, startup_snapshot: crate::js::deno_isolate_init(), - create_params: None, + create_params: create_isolate_create_params(), unsafely_ignore_certificate_errors: shared .options .unsafely_ignore_certificate_errors .clone(), - root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), seed: shared.options.seed, format_js_error_fn: Some(Arc::new(format_js_error)), create_web_worker_cb, @@ -614,28 +612,18 @@ impl CliMainWorkerFactory { should_break_on_first_statement: shared.options.inspect_brk, should_wait_for_inspector_session: shared.options.inspect_wait, strace_ops: shared.options.strace_ops.clone(), - module_loader, - fs: shared.fs.clone(), - node_services: Some(shared.create_node_init_services()), get_error_class_fn: Some(&errors::get_error_class_name), cache_storage_dir, origin_storage_dir, - blob_store: shared.blob_store.clone(), - broadcast_channel: shared.broadcast_channel.clone(), - shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()), - compiled_wasm_module_store: Some( - shared.compiled_wasm_module_store.clone(), - ), stdio, - feature_checker, - permission_desc_parser: shared.permission_desc_parser.clone(), skip_op_registration: shared.options.skip_op_registration, - v8_code_cache: shared.code_cache.clone(), + enable_stack_trace_arg_in_ops: crate::args::has_trace_permissions_enabled( + ), }; let mut worker = MainWorker::bootstrap_from_options( main_module.clone(), - permissions, + services, options, ); @@ -658,7 +646,6 @@ impl CliMainWorkerFactory { Ok(CliMainWorker { main_module, - is_main_cjs, worker, shared: shared.clone(), }) @@ -668,19 +655,19 @@ impl CliMainWorkerFactory { &self, package_folder: &Path, sub_path: Option<&str>, - ) -> Result { + ) -> Result { match self .shared .node_resolver .resolve_binary_export(package_folder, sub_path) { - Ok(node_resolution) => Ok(node_resolution), + Ok(specifier) => Ok(specifier), Err(original_err) => { // if the binary entrypoint was not found, fallback to regular node resolution let result = self.resolve_binary_entrypoint_fallback(package_folder, sub_path); match result { - Ok(Some(resolution)) => Ok(resolution), + Ok(Some(specifier)) => Ok(specifier), Ok(None) => Err(original_err.into()), Err(fallback_err) => { bail!("{:#}\n\nFallback failed: {:#}", original_err, fallback_err) @@ -695,7 +682,7 @@ impl CliMainWorkerFactory { &self, package_folder: &Path, sub_path: Option<&str>, - ) -> Result, AnyError> { + ) -> Result, AnyError> { // only fallback if the user specified a sub path if sub_path.is_none() { // it's confusing to users if the package doesn't have any binary @@ -704,28 +691,24 @@ impl CliMainWorkerFactory { return Ok(None); } - let resolution = self + let specifier = self .shared .node_resolver .resolve_package_subpath_from_deno_module( package_folder, sub_path, /* referrer */ None, - NodeResolutionMode::Execution, + ResolutionMode::Import, + NodeResolutionKind::Execution, )?; - match &resolution { - NodeResolution::BuiltIn(_) => Ok(None), - NodeResolution::CommonJs(specifier) | NodeResolution::Esm(specifier) => { - if specifier - .to_file_path() - .map(|p| p.exists()) - .unwrap_or(false) - { - Ok(Some(resolution)) - } else { - bail!("Cannot find module '{}'", specifier) - } - } + if specifier + .to_file_path() + .map(|p| p.exists()) + .unwrap_or(false) + { + Ok(Some(specifier)) + } else { + bail!("Cannot find module '{}'", specifier) } } } @@ -737,11 +720,13 @@ fn create_web_worker_callback( Arc::new(move |args| { let maybe_inspector_server = shared.maybe_inspector_server.clone(); - let ModuleLoaderAndSourceMapGetter { module_loader } = - shared.module_loader_factory.create_for_worker( - args.parent_permissions.clone(), - args.permissions.clone(), - ); + let CreateModuleLoaderResult { + module_loader, + node_require_loader, + } = shared.module_loader_factory.create_for_worker( + args.parent_permissions.clone(), + args.permissions.clone(), + ); let create_web_worker_cb = create_web_worker_callback(shared.clone(), stdio.clone()); @@ -750,10 +735,7 @@ fn create_web_worker_callback( .resolve_storage_key(&args.main_module); let cache_storage_dir = maybe_storage_key.map(|key| { // TODO(@satyarohith): storage quota management - // Note: we currently use temp_dir() to avoid managing storage size. - std::env::temp_dir() - .join("deno_cache") - .join(checksum::gen(&[key.as_bytes()])) + get_cache_storage_dir().join(checksum::gen(&[key.as_bytes()])) }); // TODO(bartlomieju): this is cruft, update FeatureChecker to spit out @@ -767,7 +749,28 @@ fn create_web_worker_callback( } } + let services = WebWorkerServiceOptions { + root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), + module_loader, + fs: shared.fs.clone(), + node_services: Some( + shared.create_node_init_services(node_require_loader), + ), + blob_store: shared.blob_store.clone(), + broadcast_channel: shared.broadcast_channel.clone(), + shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()), + compiled_wasm_module_store: Some( + shared.compiled_wasm_module_store.clone(), + ), + maybe_inspector_server, + feature_checker, + npm_process_state_provider: Some(shared.npm_process_state_provider()), + permissions: args.permissions, + }; let options = WebWorkerOptions { + name: args.name, + main_module: args.main_module.clone(), + worker_id: args.worker_id, bootstrap: BootstrapOptions { deno_version: crate::version::DENO_VERSION_INFO.deno.to_string(), args: shared.options.argv.clone(), @@ -778,7 +781,7 @@ fn create_web_worker_callback( enable_op_summary_metrics: shared.options.enable_op_summary_metrics, enable_testing_features: shared.options.enable_testing_features, locale: deno_core::v8::icu::get_language_tag(), - location: Some(args.main_module.clone()), + location: Some(args.main_module), no_color: !colors::use_color(), color_level: colors::get_color_level(), is_stdout_tty: deno_terminal::is_stdout_tty(), @@ -793,45 +796,41 @@ fn create_web_worker_callback( mode: WorkerExecutionMode::Worker, serve_port: shared.options.serve_port, serve_host: shared.options.serve_host.clone(), + otel_config: shared.otel_config.clone(), }, extensions: vec![], startup_snapshot: crate::js::deno_isolate_init(), + create_params: create_isolate_create_params(), unsafely_ignore_certificate_errors: shared .options .unsafely_ignore_certificate_errors .clone(), - root_cert_store_provider: Some(shared.root_cert_store_provider.clone()), seed: shared.options.seed, create_web_worker_cb, format_js_error_fn: Some(Arc::new(format_js_error)), - module_loader, - fs: shared.fs.clone(), - node_services: Some(shared.create_node_init_services()), worker_type: args.worker_type, - maybe_inspector_server, get_error_class_fn: Some(&errors::get_error_class_name), - blob_store: shared.blob_store.clone(), - broadcast_channel: shared.broadcast_channel.clone(), - shared_array_buffer_store: Some(shared.shared_array_buffer_store.clone()), - compiled_wasm_module_store: Some( - shared.compiled_wasm_module_store.clone(), - ), stdio: stdio.clone(), cache_storage_dir, - feature_checker, - permission_desc_parser: shared.permission_desc_parser.clone(), strace_ops: shared.options.strace_ops.clone(), close_on_idle: args.close_on_idle, maybe_worker_metadata: args.maybe_worker_metadata, + enable_stack_trace_arg_in_ops: crate::args::has_trace_permissions_enabled( + ), }; - WebWorker::bootstrap_from_options( - args.name, - args.permissions, - args.main_module, - args.worker_id, - options, - ) + WebWorker::bootstrap_from_options(services, options) + }) +} + +/// By default V8 uses 1.4Gb heap limit which is meant for browser tabs. +/// Instead probe for the total memory on the system and use it instead +/// as a default. +pub fn create_isolate_create_params() -> Option { + let maybe_mem_info = deno_runtime::sys_info::mem_info(); + maybe_mem_info.map(|mem_info| { + v8::CreateParams::default() + .heap_limits_from_system_memory(mem_info.total, 0) }) } @@ -841,23 +840,44 @@ fn create_web_worker_callback( mod tests { use super::*; use deno_core::resolve_path; + use deno_core::FsModuleLoader; use deno_fs::RealFs; use deno_runtime::deno_permissions::Permissions; + use deno_runtime::permissions::RuntimePermissionDescriptorParser; fn create_test_worker() -> MainWorker { let main_module = resolve_path("./hello.js", &std::env::current_dir().unwrap()).unwrap(); - let permissions = PermissionsContainer::new( - Arc::new(RuntimePermissionDescriptorParser::new(Arc::new(RealFs))), - Permissions::none_without_prompt(), - ); - + let fs = Arc::new(RealFs); + let permission_desc_parser = + Arc::new(RuntimePermissionDescriptorParser::new(fs.clone())); let options = WorkerOptions { startup_snapshot: crate::js::deno_isolate_init(), ..Default::default() }; - MainWorker::bootstrap_from_options(main_module, permissions, options) + MainWorker::bootstrap_from_options( + main_module, + WorkerServiceOptions { + module_loader: Rc::new(FsModuleLoader), + permissions: PermissionsContainer::new( + permission_desc_parser, + Permissions::none_without_prompt(), + ), + blob_store: Default::default(), + broadcast_channel: Default::default(), + feature_checker: Default::default(), + node_services: Default::default(), + npm_process_state_provider: Default::default(), + root_cert_store_provider: Default::default(), + fetch_dns_resolver: Default::default(), + shared_array_buffer_store: Default::default(), + compiled_wasm_module_store: Default::default(), + v8_code_cache: Default::default(), + fs, + }, + options, + ) } #[tokio::test] diff --git a/ext/broadcast_channel/Cargo.toml b/ext/broadcast_channel/Cargo.toml index 8b8c9a18838f99..a0383592876727 100644 --- a/ext/broadcast_channel/Cargo.toml +++ b/ext/broadcast_channel/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_broadcast_channel" -version = "0.162.0" +version = "0.174.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -16,5 +16,6 @@ path = "lib.rs" [dependencies] async-trait.workspace = true deno_core.workspace = true +thiserror.workspace = true tokio.workspace = true uuid.workspace = true diff --git a/ext/broadcast_channel/in_memory_broadcast_channel.rs b/ext/broadcast_channel/in_memory_broadcast_channel.rs index 00b52a9d60bf77..61dc68e17de88a 100644 --- a/ext/broadcast_channel/in_memory_broadcast_channel.rs +++ b/ext/broadcast_channel/in_memory_broadcast_channel.rs @@ -3,13 +3,13 @@ use std::sync::Arc; use async_trait::async_trait; -use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use tokio::sync::broadcast; use tokio::sync::mpsc; use uuid::Uuid; use crate::BroadcastChannel; +use crate::BroadcastChannelError; #[derive(Clone)] pub struct InMemoryBroadcastChannel(Arc>>); @@ -41,7 +41,7 @@ impl Default for InMemoryBroadcastChannel { impl BroadcastChannel for InMemoryBroadcastChannel { type Resource = InMemoryBroadcastChannelResource; - fn subscribe(&self) -> Result { + fn subscribe(&self) -> Result { let (cancel_tx, cancel_rx) = mpsc::unbounded_channel(); let broadcast_rx = self.0.lock().subscribe(); let rx = tokio::sync::Mutex::new((broadcast_rx, cancel_rx)); @@ -53,7 +53,10 @@ impl BroadcastChannel for InMemoryBroadcastChannel { }) } - fn unsubscribe(&self, resource: &Self::Resource) -> Result<(), AnyError> { + fn unsubscribe( + &self, + resource: &Self::Resource, + ) -> Result<(), BroadcastChannelError> { Ok(resource.cancel_tx.send(())?) } @@ -62,7 +65,7 @@ impl BroadcastChannel for InMemoryBroadcastChannel { resource: &Self::Resource, name: String, data: Vec, - ) -> Result<(), AnyError> { + ) -> Result<(), BroadcastChannelError> { let name = Arc::new(name); let data = Arc::new(data); let uuid = resource.uuid; @@ -73,7 +76,7 @@ impl BroadcastChannel for InMemoryBroadcastChannel { async fn recv( &self, resource: &Self::Resource, - ) -> Result, AnyError> { + ) -> Result, BroadcastChannelError> { let mut g = resource.rx.lock().await; let (broadcast_rx, cancel_rx) = &mut *g; loop { diff --git a/ext/broadcast_channel/lib.rs b/ext/broadcast_channel/lib.rs index 47c48656d87b56..c1de118a364fad 100644 --- a/ext/broadcast_channel/lib.rs +++ b/ext/broadcast_channel/lib.rs @@ -10,34 +10,69 @@ use std::path::PathBuf; use std::rc::Rc; use async_trait::async_trait; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::JsBuffer; use deno_core::OpState; use deno_core::Resource; use deno_core::ResourceId; +use tokio::sync::broadcast::error::SendError as BroadcastSendError; +use tokio::sync::mpsc::error::SendError as MpscSendError; pub const UNSTABLE_FEATURE_NAME: &str = "broadcast-channel"; +#[derive(Debug, thiserror::Error)] +pub enum BroadcastChannelError { + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error(transparent)] + MPSCSendError(MpscSendError>), + #[error(transparent)] + BroadcastSendError( + BroadcastSendError>, + ), + #[error(transparent)] + Other(deno_core::error::AnyError), +} + +impl From> + for BroadcastChannelError +{ + fn from(value: MpscSendError) -> Self { + BroadcastChannelError::MPSCSendError(MpscSendError(Box::new(value.0))) + } +} +impl From> + for BroadcastChannelError +{ + fn from(value: BroadcastSendError) -> Self { + BroadcastChannelError::BroadcastSendError(BroadcastSendError(Box::new( + value.0, + ))) + } +} + #[async_trait] pub trait BroadcastChannel: Clone { type Resource: Resource; - fn subscribe(&self) -> Result; + fn subscribe(&self) -> Result; - fn unsubscribe(&self, resource: &Self::Resource) -> Result<(), AnyError>; + fn unsubscribe( + &self, + resource: &Self::Resource, + ) -> Result<(), BroadcastChannelError>; async fn send( &self, resource: &Self::Resource, name: String, data: Vec, - ) -> Result<(), AnyError>; + ) -> Result<(), BroadcastChannelError>; async fn recv( &self, resource: &Self::Resource, - ) -> Result, AnyError>; + ) -> Result, BroadcastChannelError>; } pub type Message = (String, Vec); @@ -46,7 +81,7 @@ pub type Message = (String, Vec); #[smi] pub fn op_broadcast_subscribe( state: &mut OpState, -) -> Result +) -> Result where BC: BroadcastChannel + 'static, { @@ -62,11 +97,14 @@ where pub fn op_broadcast_unsubscribe( state: &mut OpState, #[smi] rid: ResourceId, -) -> Result<(), AnyError> +) -> Result<(), BroadcastChannelError> where BC: BroadcastChannel + 'static, { - let resource = state.resource_table.get::(rid)?; + let resource = state + .resource_table + .get::(rid) + .map_err(BroadcastChannelError::Resource)?; let bc = state.borrow::(); bc.unsubscribe(&resource) } @@ -77,11 +115,15 @@ pub async fn op_broadcast_send( #[smi] rid: ResourceId, #[string] name: String, #[buffer] buf: JsBuffer, -) -> Result<(), AnyError> +) -> Result<(), BroadcastChannelError> where BC: BroadcastChannel + 'static, { - let resource = state.borrow().resource_table.get::(rid)?; + let resource = state + .borrow() + .resource_table + .get::(rid) + .map_err(BroadcastChannelError::Resource)?; let bc = state.borrow().borrow::().clone(); bc.send(&resource, name, buf.to_vec()).await } @@ -91,11 +133,15 @@ where pub async fn op_broadcast_recv( state: Rc>, #[smi] rid: ResourceId, -) -> Result, AnyError> +) -> Result, BroadcastChannelError> where BC: BroadcastChannel + 'static, { - let resource = state.borrow().resource_table.get::(rid)?; + let resource = state + .borrow() + .resource_table + .get::(rid) + .map_err(BroadcastChannelError::Resource)?; let bc = state.borrow().borrow::().clone(); bc.recv(&resource).await } diff --git a/ext/cache/Cargo.toml b/ext/cache/Cargo.toml index 5c6f162f17e047..edaf443c0572c9 100644 --- a/ext/cache/Cargo.toml +++ b/ext/cache/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_cache" -version = "0.100.0" +version = "0.112.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -19,4 +19,5 @@ deno_core.workspace = true rusqlite.workspace = true serde.workspace = true sha2.workspace = true +thiserror.workspace = true tokio.workspace = true diff --git a/ext/cache/lib.rs b/ext/cache/lib.rs index f6d758b95c0143..524d4cea0574ab 100644 --- a/ext/cache/lib.rs +++ b/ext/cache/lib.rs @@ -7,7 +7,6 @@ use std::sync::Arc; use async_trait::async_trait; use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::serde::Deserialize; use deno_core::serde::Serialize; @@ -19,8 +18,24 @@ use deno_core::ResourceId; mod sqlite; pub use sqlite::SqliteBackedCache; +#[derive(Debug, thiserror::Error)] +pub enum CacheError { + #[error(transparent)] + Sqlite(#[from] rusqlite::Error), + #[error(transparent)] + JoinError(#[from] tokio::task::JoinError), + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error(transparent)] + Other(deno_core::error::AnyError), + #[error("{0}")] + Io(#[from] std::io::Error), +} + #[derive(Clone)] -pub struct CreateCache(pub Arc C>); +pub struct CreateCache( + pub Arc Result>, +); deno_core::extension!(deno_cache, deps = [ deno_webidl, deno_web, deno_url, deno_fetch ], @@ -92,26 +107,31 @@ pub struct CacheDeleteRequest { pub trait Cache: Clone + 'static { type CacheMatchResourceType: Resource; - async fn storage_open(&self, cache_name: String) -> Result; - async fn storage_has(&self, cache_name: String) -> Result; - async fn storage_delete(&self, cache_name: String) -> Result; + async fn storage_open(&self, cache_name: String) -> Result; + async fn storage_has(&self, cache_name: String) -> Result; + async fn storage_delete( + &self, + cache_name: String, + ) -> Result; /// Put a resource into the cache. async fn put( &self, request_response: CachePutRequest, resource: Option>, - ) -> Result<(), AnyError>; + ) -> Result<(), CacheError>; async fn r#match( &self, request: CacheMatchRequest, ) -> Result< Option<(CacheMatchResponseMeta, Option)>, - AnyError, + CacheError, >; - async fn delete(&self, request: CacheDeleteRequest) - -> Result; + async fn delete( + &self, + request: CacheDeleteRequest, + ) -> Result; } #[op2(async)] @@ -119,7 +139,7 @@ pub trait Cache: Clone + 'static { pub async fn op_cache_storage_open( state: Rc>, #[string] cache_name: String, -) -> Result +) -> Result where CA: Cache, { @@ -131,7 +151,7 @@ where pub async fn op_cache_storage_has( state: Rc>, #[string] cache_name: String, -) -> Result +) -> Result where CA: Cache, { @@ -143,7 +163,7 @@ where pub async fn op_cache_storage_delete( state: Rc>, #[string] cache_name: String, -) -> Result +) -> Result where CA: Cache, { @@ -155,13 +175,19 @@ where pub async fn op_cache_put( state: Rc>, #[serde] request_response: CachePutRequest, -) -> Result<(), AnyError> +) -> Result<(), CacheError> where CA: Cache, { let cache = get_cache::(&state)?; let resource = match request_response.response_rid { - Some(rid) => Some(state.borrow_mut().resource_table.take_any(rid)?), + Some(rid) => Some( + state + .borrow_mut() + .resource_table + .take_any(rid) + .map_err(CacheError::Resource)?, + ), None => None, }; cache.put(request_response, resource).await @@ -172,7 +198,7 @@ where pub async fn op_cache_match( state: Rc>, #[serde] request: CacheMatchRequest, -) -> Result, AnyError> +) -> Result, CacheError> where CA: Cache, { @@ -191,7 +217,7 @@ where pub async fn op_cache_delete( state: Rc>, #[serde] request: CacheDeleteRequest, -) -> Result +) -> Result where CA: Cache, { @@ -199,7 +225,7 @@ where cache.delete(request).await } -pub fn get_cache(state: &Rc>) -> Result +pub fn get_cache(state: &Rc>) -> Result where CA: Cache, { @@ -207,11 +233,13 @@ where if let Some(cache) = state.try_borrow::() { Ok(cache.clone()) } else if let Some(create_cache) = state.try_borrow::>() { - let cache = create_cache.0(); + let cache = create_cache.0()?; state.put(cache); Ok(state.borrow::().clone()) } else { - Err(type_error("CacheStorage is not available in this context")) + Err(CacheError::Other(type_error( + "CacheStorage is not available in this context", + ))) } } diff --git a/ext/cache/sqlite.rs b/ext/cache/sqlite.rs index c3c55dd5e9b4ce..469e3e51d6f313 100644 --- a/ext/cache/sqlite.rs +++ b/ext/cache/sqlite.rs @@ -8,6 +8,7 @@ use std::time::SystemTime; use std::time::UNIX_EPOCH; use async_trait::async_trait; +use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::futures::future::poll_fn; use deno_core::parking_lot::Mutex; @@ -30,6 +31,7 @@ use crate::serialize_headers; use crate::vary_header_matches; use crate::Cache; use crate::CacheDeleteRequest; +use crate::CacheError; use crate::CacheMatchRequest; use crate::CacheMatchResponseMeta; use crate::CachePutRequest; @@ -41,10 +43,16 @@ pub struct SqliteBackedCache { } impl SqliteBackedCache { - pub fn new(cache_storage_dir: PathBuf) -> Self { + pub fn new(cache_storage_dir: PathBuf) -> Result { { std::fs::create_dir_all(&cache_storage_dir) - .expect("failed to create cache dir"); + .with_context(|| { + format!( + "Failed to create cache storage directory {}", + cache_storage_dir.display() + ) + }) + .map_err(CacheError::Other)?; let path = cache_storage_dir.join("cache_metadata.db"); let connection = rusqlite::Connection::open(&path).unwrap_or_else(|_| { panic!("failed to open cache db at {}", path.display()) @@ -56,18 +64,14 @@ impl SqliteBackedCache { PRAGMA synchronous=NORMAL; PRAGMA optimize; "; - connection - .execute_batch(initial_pragmas) - .expect("failed to execute pragmas"); - connection - .execute( - "CREATE TABLE IF NOT EXISTS cache_storage ( + connection.execute_batch(initial_pragmas)?; + connection.execute( + "CREATE TABLE IF NOT EXISTS cache_storage ( id INTEGER PRIMARY KEY, cache_name TEXT NOT NULL UNIQUE )", - (), - ) - .expect("failed to create cache_storage table"); + (), + )?; connection .execute( "CREATE TABLE IF NOT EXISTS request_response_list ( @@ -85,12 +89,11 @@ impl SqliteBackedCache { UNIQUE (cache_id, request_url) )", (), - ) - .expect("failed to create request_response_list table"); - SqliteBackedCache { + )?; + Ok(SqliteBackedCache { connection: Arc::new(Mutex::new(connection)), cache_storage_dir, - } + }) } } } @@ -102,7 +105,7 @@ impl Cache for SqliteBackedCache { /// Open a cache storage. Internally, this creates a row in the /// sqlite db if the cache doesn't exist and returns the internal id /// of the cache. - async fn storage_open(&self, cache_name: String) -> Result { + async fn storage_open(&self, cache_name: String) -> Result { let db = self.connection.clone(); let cache_storage_dir = self.cache_storage_dir.clone(); spawn_blocking(move || { @@ -121,14 +124,14 @@ impl Cache for SqliteBackedCache { )?; let responses_dir = get_responses_dir(cache_storage_dir, cache_id); std::fs::create_dir_all(responses_dir)?; - Ok::(cache_id) + Ok::(cache_id) }) .await? } /// Check if a cache with the provided name exists. /// Note: this doesn't check the disk, it only checks the sqlite db. - async fn storage_has(&self, cache_name: String) -> Result { + async fn storage_has(&self, cache_name: String) -> Result { let db = self.connection.clone(); spawn_blocking(move || { let db = db.lock(); @@ -140,13 +143,16 @@ impl Cache for SqliteBackedCache { Ok(count > 0) }, )?; - Ok::(cache_exists) + Ok::(cache_exists) }) .await? } /// Delete a cache storage. Internally, this deletes the row in the sqlite db. - async fn storage_delete(&self, cache_name: String) -> Result { + async fn storage_delete( + &self, + cache_name: String, + ) -> Result { let db = self.connection.clone(); let cache_storage_dir = self.cache_storage_dir.clone(); spawn_blocking(move || { @@ -167,7 +173,7 @@ impl Cache for SqliteBackedCache { std::fs::remove_dir_all(cache_dir)?; } } - Ok::(maybe_cache_id.is_some()) + Ok::(maybe_cache_id.is_some()) }) .await? } @@ -176,10 +182,12 @@ impl Cache for SqliteBackedCache { &self, request_response: CachePutRequest, resource: Option>, - ) -> Result<(), AnyError> { + ) -> Result<(), CacheError> { let db = self.connection.clone(); let cache_storage_dir = self.cache_storage_dir.clone(); - let now = SystemTime::now().duration_since(UNIX_EPOCH)?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("SystemTime is before unix epoch"); if let Some(resource) = resource { let body_key = hash(&format!( @@ -193,7 +201,11 @@ impl Cache for SqliteBackedCache { let mut file = tokio::fs::File::create(response_path).await?; let mut buf = BufMutView::new(64 * 1024); loop { - let (size, buf2) = resource.clone().read_byob(buf).await?; + let (size, buf2) = resource + .clone() + .read_byob(buf) + .await + .map_err(CacheError::Other)?; if size == 0 { break; } @@ -224,7 +236,7 @@ impl Cache for SqliteBackedCache { request: CacheMatchRequest, ) -> Result< Option<(CacheMatchResponseMeta, Option)>, - AnyError, + CacheError, > { let db = self.connection.clone(); let cache_storage_dir = self.cache_storage_dir.clone(); @@ -290,19 +302,17 @@ impl Cache for SqliteBackedCache { } Err(err) => return Err(err.into()), }; - return Ok(Some((cache_meta, Some(CacheResponseResource::new(file))))); - } - Some((cache_meta, None)) => { - return Ok(Some((cache_meta, None))); + Ok(Some((cache_meta, Some(CacheResponseResource::new(file))))) } - None => return Ok(None), + Some((cache_meta, None)) => Ok(Some((cache_meta, None))), + None => Ok(None), } } async fn delete( &self, request: CacheDeleteRequest, - ) -> Result { + ) -> Result { let db = self.connection.clone(); spawn_blocking(move || { // TODO(@satyarohith): remove the response body from disk if one exists @@ -311,17 +321,17 @@ impl Cache for SqliteBackedCache { "DELETE FROM request_response_list WHERE cache_id = ?1 AND request_url = ?2", (request.cache_id, &request.request_url), )?; - Ok::(rows_effected > 0) + Ok::(rows_effected > 0) }) .await? } } async fn insert_cache_asset( - db: Arc>, + db: Arc>, put: CachePutRequest, response_body_key: Option, -) -> Result, deno_core::anyhow::Error> { +) -> Result, CacheError> { spawn_blocking(move || { let maybe_response_body = { let db = db.lock(); @@ -339,7 +349,7 @@ async fn insert_cache_asset( response_body_key, put.response_status, put.response_status_text, - SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(), + SystemTime::now().duration_since(UNIX_EPOCH).expect("SystemTime is before unix epoch").as_secs(), ), |row| { let response_body_key: Option = row.get(0)?; @@ -347,7 +357,7 @@ async fn insert_cache_asset( }, )? }; - Ok::, AnyError>(maybe_response_body) + Ok::, CacheError>(maybe_response_body) }).await? } diff --git a/ext/canvas/Cargo.toml b/ext/canvas/Cargo.toml index 96881682a4ac88..db624670f38942 100644 --- a/ext/canvas/Cargo.toml +++ b/ext/canvas/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_canvas" -version = "0.37.0" +version = "0.49.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -18,3 +18,4 @@ deno_core.workspace = true deno_webgpu.workspace = true image = { version = "0.24.7", default-features = false, features = ["png"] } serde = { workspace = true, features = ["derive"] } +thiserror.workspace = true diff --git a/ext/canvas/lib.rs b/ext/canvas/lib.rs index 72173f133158ba..defb288ac9d2dd 100644 --- a/ext/canvas/lib.rs +++ b/ext/canvas/lib.rs @@ -1,7 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::ToJsBuffer; use image::imageops::FilterType; @@ -13,6 +11,14 @@ use serde::Deserialize; use serde::Serialize; use std::path::PathBuf; +#[derive(Debug, thiserror::Error)] +pub enum CanvasError { + #[error("Color type '{0:?}' not supported")] + UnsupportedColorType(ColorType), + #[error(transparent)] + Image(#[from] image::ImageError), +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "snake_case")] enum ImageResizeQuality { @@ -43,7 +49,7 @@ struct ImageProcessArgs { fn op_image_process( #[buffer] buf: &[u8], #[serde] args: ImageProcessArgs, -) -> Result { +) -> ToJsBuffer { let view = RgbaImage::from_vec(args.width, args.height, buf.to_vec()).unwrap(); @@ -105,7 +111,7 @@ fn op_image_process( } } - Ok(image_out.to_vec().into()) + image_out.to_vec().into() } #[derive(Debug, Serialize)] @@ -117,17 +123,16 @@ struct DecodedPng { #[op2] #[serde] -fn op_image_decode_png(#[buffer] buf: &[u8]) -> Result { +fn op_image_decode_png( + #[buffer] buf: &[u8], +) -> Result { let png = image::codecs::png::PngDecoder::new(buf)?; let (width, height) = png.dimensions(); // TODO(@crowlKats): maybe use DynamicImage https://docs.rs/image/0.24.7/image/enum.DynamicImage.html ? if png.color_type() != ColorType::Rgba8 { - return Err(type_error(format!( - "Color type '{:?}' not supported", - png.color_type() - ))); + return Err(CanvasError::UnsupportedColorType(png.color_type())); } // read_image will assert that the buffer is the correct size, so we need to fill it with zeros diff --git a/ext/console/01_console.js b/ext/console/01_console.js index 1ca23d5a49aff5..3803492b90d490 100644 --- a/ext/console/01_console.js +++ b/ext/console/01_console.js @@ -84,6 +84,7 @@ const { NumberIsInteger, NumberIsNaN, NumberParseInt, + NumberParseFloat, NumberPrototypeToFixed, NumberPrototypeToString, NumberPrototypeValueOf, @@ -1301,7 +1302,9 @@ function getKeys(value, showHidden) { ArrayPrototypePushApply(keys, ArrayPrototypeFilter(symbols, filter)); } } - keys = ArrayPrototypeFilter(keys, (key) => key !== "cause"); + if (ObjectPrototypeIsPrototypeOf(ErrorPrototype, value)) { + keys = ArrayPrototypeFilter(keys, (key) => key !== "cause"); + } return keys; } @@ -2650,6 +2653,7 @@ const HSL_PATTERN = new SafeRegExp( ); function parseCssColor(colorString) { + colorString = StringPrototypeToLowerCase(colorString); if (colorKeywords.has(colorString)) { colorString = colorKeywords.get(colorString); } @@ -3008,20 +3012,18 @@ function inspectArgs(args, inspectOptions = { __proto__: null }) { } else if (ArrayPrototypeIncludes(["d", "i"], char)) { // Format as an integer. const value = args[a++]; - if (typeof value == "bigint") { - formattedArg = `${value}n`; - } else if (typeof value == "number") { - formattedArg = `${NumberParseInt(String(value))}`; - } else { + if (typeof value === "symbol") { formattedArg = "NaN"; + } else { + formattedArg = `${NumberParseInt(value)}`; } } else if (char == "f") { // Format as a floating point value. const value = args[a++]; - if (typeof value == "number") { - formattedArg = `${value}`; - } else { + if (typeof value === "symbol") { formattedArg = "NaN"; + } else { + formattedArg = `${NumberParseFloat(value)}`; } } else if (ArrayPrototypeIncludes(["O", "o"], char)) { // Format as an object. @@ -3255,7 +3257,7 @@ class Console { const stringifyValue = (value) => inspectValueWithQuotes(value, { - ...getDefaultInspectOptions(), + ...getConsoleInspectOptions(noColorStdout()), depth: 1, compact: true, }); diff --git a/ext/console/Cargo.toml b/ext/console/Cargo.toml index ef64398584e266..49ad2434563782 100644 --- a/ext/console/Cargo.toml +++ b/ext/console/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_console" -version = "0.168.0" +version = "0.180.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/console/internal.d.ts b/ext/console/internal.d.ts index 45af616d673b45..5f9627cf56ef41 100644 --- a/ext/console/internal.d.ts +++ b/ext/console/internal.d.ts @@ -9,4 +9,7 @@ declare module "ext:deno_console/01_console.js" { keys: (keyof TObject)[]; evaluate: boolean; }): Record; + + class Console { + } } diff --git a/ext/cron/Cargo.toml b/ext/cron/Cargo.toml index 966ac450e9b4c8..d8f7f20d0c9f6e 100644 --- a/ext/cron/Cargo.toml +++ b/ext/cron/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_cron" -version = "0.48.0" +version = "0.60.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -19,4 +19,5 @@ async-trait.workspace = true chrono = { workspace = true, features = ["now"] } deno_core.workspace = true saffron.workspace = true +thiserror.workspace = true tokio.workspace = true diff --git a/ext/cron/interface.rs b/ext/cron/interface.rs index 01b1d178951b2b..a19525cc4e2539 100644 --- a/ext/cron/interface.rs +++ b/ext/cron/interface.rs @@ -1,17 +1,17 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use crate::CronError; use async_trait::async_trait; -use deno_core::error::AnyError; pub trait CronHandler { type EH: CronHandle + 'static; - fn create(&self, spec: CronSpec) -> Result; + fn create(&self, spec: CronSpec) -> Result; } #[async_trait(?Send)] pub trait CronHandle { - async fn next(&self, prev_success: bool) -> Result; + async fn next(&self, prev_success: bool) -> Result; fn close(&self); } diff --git a/ext/cron/lib.rs b/ext/cron/lib.rs index e350e4d698c115..feffb5e5112114 100644 --- a/ext/cron/lib.rs +++ b/ext/cron/lib.rs @@ -7,16 +7,13 @@ use std::borrow::Cow; use std::cell::RefCell; use std::rc::Rc; +pub use crate::interface::*; use deno_core::error::get_custom_error_class; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; use deno_core::Resource; use deno_core::ResourceId; -pub use crate::interface::*; - pub const UNSTABLE_FEATURE_NAME: &str = "cron"; deno_core::extension!(deno_cron, @@ -49,6 +46,28 @@ impl Resource for CronResource { } } +#[derive(Debug, thiserror::Error)] +pub enum CronError { + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error("Cron name cannot exceed 64 characters: current length {0}")] + NameExceeded(usize), + #[error("Invalid cron name: only alphanumeric characters, whitespace, hyphens, and underscores are allowed")] + NameInvalid, + #[error("Cron with this name already exists")] + AlreadyExists, + #[error("Too many crons")] + TooManyCrons, + #[error("Invalid cron schedule")] + InvalidCron, + #[error("Invalid backoff schedule")] + InvalidBackoff, + #[error(transparent)] + AcquireError(#[from] tokio::sync::AcquireError), + #[error(transparent)] + Other(deno_core::error::AnyError), +} + #[op2] #[smi] fn op_cron_create( @@ -56,7 +75,7 @@ fn op_cron_create( #[string] name: String, #[string] cron_schedule: String, #[serde] backoff_schedule: Option>, -) -> Result +) -> Result where C: CronHandler + 'static, { @@ -90,7 +109,7 @@ async fn op_cron_next( state: Rc>, #[smi] rid: ResourceId, prev_success: bool, -) -> Result +) -> Result where C: CronHandler + 'static, { @@ -102,7 +121,7 @@ where if get_custom_error_class(&err) == Some("BadResource") { return Ok(false); } else { - return Err(err); + return Err(CronError::Resource(err)); } } }; @@ -112,17 +131,14 @@ where cron_handler.next(prev_success).await } -fn validate_cron_name(name: &str) -> Result<(), AnyError> { +fn validate_cron_name(name: &str) -> Result<(), CronError> { if name.len() > 64 { - return Err(type_error(format!( - "Cron name cannot exceed 64 characters: current length {}", - name.len() - ))); + return Err(CronError::NameExceeded(name.len())); } if !name.chars().all(|c| { c.is_ascii_whitespace() || c.is_ascii_alphanumeric() || c == '_' || c == '-' }) { - return Err(type_error("Invalid cron name: only alphanumeric characters, whitespace, hyphens, and underscores are allowed")); + return Err(CronError::NameInvalid); } Ok(()) } diff --git a/ext/cron/local.rs b/ext/cron/local.rs index dd60e750a07408..1110baadb84d0e 100644 --- a/ext/cron/local.rs +++ b/ext/cron/local.rs @@ -10,8 +10,6 @@ use std::rc::Weak; use std::sync::Arc; use async_trait::async_trait; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::futures; use deno_core::futures::FutureExt; use deno_core::unsync::spawn; @@ -21,6 +19,7 @@ use tokio::sync::mpsc::WeakSender; use tokio::sync::OwnedSemaphorePermit; use tokio::sync::Semaphore; +use crate::CronError; use crate::CronHandle; use crate::CronHandler; use crate::CronSpec; @@ -81,7 +80,7 @@ impl LocalCronHandler { async fn cron_loop( runtime_state: Rc>, mut cron_schedule_rx: mpsc::Receiver<(String, bool)>, - ) -> Result<(), AnyError> { + ) -> Result<(), CronError> { loop { let earliest_deadline = runtime_state .borrow() @@ -154,7 +153,7 @@ impl LocalCronHandler { impl RuntimeState { fn get_ready_crons( &mut self, - ) -> Result)>, AnyError> { + ) -> Result)>, CronError> { let now = chrono::Utc::now().timestamp_millis() as u64; let ready = { @@ -191,7 +190,7 @@ impl RuntimeState { impl CronHandler for LocalCronHandler { type EH = CronExecutionHandle; - fn create(&self, spec: CronSpec) -> Result { + fn create(&self, spec: CronSpec) -> Result { // Ensure that the cron loop is started. self.cron_loop_join_handle.get_or_init(|| { let (cron_schedule_tx, cron_schedule_rx) = @@ -208,17 +207,17 @@ impl CronHandler for LocalCronHandler { let mut runtime_state = self.runtime_state.borrow_mut(); if runtime_state.crons.len() > MAX_CRONS { - return Err(type_error("Too many crons")); + return Err(CronError::TooManyCrons); } if runtime_state.crons.contains_key(&spec.name) { - return Err(type_error("Cron with this name already exists")); + return Err(CronError::AlreadyExists); } // Validate schedule expression. spec .cron_schedule .parse::() - .map_err(|_| type_error("Invalid cron schedule"))?; + .map_err(|_| CronError::InvalidCron)?; // Validate backoff_schedule. if let Some(backoff_schedule) = &spec.backoff_schedule { @@ -263,7 +262,7 @@ struct Inner { #[async_trait(?Send)] impl CronHandle for CronExecutionHandle { - async fn next(&self, prev_success: bool) -> Result { + async fn next(&self, prev_success: bool) -> Result { self.inner.borrow_mut().permit.take(); if self @@ -300,7 +299,7 @@ impl CronHandle for CronExecutionHandle { } } -fn compute_next_deadline(cron_expression: &str) -> Result { +fn compute_next_deadline(cron_expression: &str) -> Result { let now = chrono::Utc::now(); if let Ok(test_schedule) = env::var("DENO_CRON_TEST_SCHEDULE_OFFSET") { @@ -311,19 +310,21 @@ fn compute_next_deadline(cron_expression: &str) -> Result { let cron = cron_expression .parse::() - .map_err(|_| anyhow::anyhow!("invalid cron expression"))?; + .map_err(|_| CronError::InvalidCron)?; let Some(next_deadline) = cron.next_after(now) else { - return Err(anyhow::anyhow!("invalid cron expression")); + return Err(CronError::InvalidCron); }; Ok(next_deadline.timestamp_millis() as u64) } -fn validate_backoff_schedule(backoff_schedule: &[u32]) -> Result<(), AnyError> { +fn validate_backoff_schedule( + backoff_schedule: &[u32], +) -> Result<(), CronError> { if backoff_schedule.len() > MAX_BACKOFF_COUNT { - return Err(type_error("Invalid backoff schedule")); + return Err(CronError::InvalidBackoff); } if backoff_schedule.iter().any(|s| *s > MAX_BACKOFF_MS) { - return Err(type_error("Invalid backoff schedule")); + return Err(CronError::InvalidBackoff); } Ok(()) } diff --git a/ext/crypto/00_crypto.js b/ext/crypto/00_crypto.js index 8e43b76f7c526d..63b1905145fa0a 100644 --- a/ext/crypto/00_crypto.js +++ b/ext/crypto/00_crypto.js @@ -18,21 +18,27 @@ import { op_crypto_decrypt, op_crypto_derive_bits, op_crypto_derive_bits_x25519, + op_crypto_derive_bits_x448, op_crypto_encrypt, op_crypto_export_key, op_crypto_export_pkcs8_ed25519, op_crypto_export_pkcs8_x25519, + op_crypto_export_pkcs8_x448, op_crypto_export_spki_ed25519, op_crypto_export_spki_x25519, + op_crypto_export_spki_x448, op_crypto_generate_ed25519_keypair, op_crypto_generate_key, op_crypto_generate_x25519_keypair, + op_crypto_generate_x448_keypair, op_crypto_get_random_values, op_crypto_import_key, op_crypto_import_pkcs8_ed25519, op_crypto_import_pkcs8_x25519, + op_crypto_import_pkcs8_x448, op_crypto_import_spki_ed25519, op_crypto_import_spki_x25519, + op_crypto_import_spki_x448, op_crypto_jwk_x_ed25519, op_crypto_random_uuid, op_crypto_sign_ed25519, @@ -134,6 +140,7 @@ const supportedAlgorithms = { "AES-KW": "AesKeyGenParams", "HMAC": "HmacKeyGenParams", "X25519": null, + "X448": null, "Ed25519": null, }, "sign": { @@ -165,12 +172,14 @@ const supportedAlgorithms = { "AES-KW": null, "Ed25519": null, "X25519": null, + "X448": null, }, "deriveBits": { "HKDF": "HkdfParams", "PBKDF2": "Pbkdf2Params", "ECDH": "EcdhKeyDeriveParams", "X25519": "EcdhKeyDeriveParams", + "X448": "EcdhKeyDeriveParams", }, "encrypt": { "RSA-OAEP": "RsaOaepParams", @@ -1037,6 +1046,10 @@ class SubtleCrypto { result = exportKeyEd25519(format, key, innerKey); break; } + case "X448": { + result = exportKeyX448(format, key, innerKey); + break; + } case "X25519": { result = exportKeyX25519(format, key, innerKey); break; @@ -1954,6 +1967,48 @@ async function generateKey(normalizedAlgorithm, extractable, usages) { return generateKeyAES(normalizedAlgorithm, extractable, usages); } + case "X448": { + if ( + ArrayPrototypeFind( + usages, + (u) => !ArrayPrototypeIncludes(["deriveKey", "deriveBits"], u), + ) !== undefined + ) { + throw new DOMException("Invalid key usage", "SyntaxError"); + } + const privateKeyData = new Uint8Array(56); + const publicKeyData = new Uint8Array(56); + + op_crypto_generate_x448_keypair(privateKeyData, publicKeyData); + + const handle = {}; + WeakMapPrototypeSet(KEY_STORE, handle, privateKeyData); + + const publicHandle = {}; + WeakMapPrototypeSet(KEY_STORE, publicHandle, publicKeyData); + + const algorithm = { + name: algorithmName, + }; + + const publicKey = constructKey( + "public", + true, + usageIntersection(usages, []), + algorithm, + publicHandle, + ); + + const privateKey = constructKey( + "private", + extractable, + usageIntersection(usages, ["deriveKey", "deriveBits"]), + algorithm, + handle, + ); + + return { publicKey, privateKey }; + } case "X25519": { if ( ArrayPrototypeFind( @@ -2100,6 +2155,211 @@ async function generateKey(normalizedAlgorithm, extractable, usages) { } } +function importKeyX448( + format, + keyData, + extractable, + keyUsages, +) { + switch (format) { + case "raw": { + // 1. + if (keyUsages.length > 0) { + throw new DOMException("Invalid key usage", "SyntaxError"); + } + + const handle = {}; + WeakMapPrototypeSet(KEY_STORE, handle, keyData); + + // 2-3. + const algorithm = { + name: "X448", + }; + + // 4-6. + return constructKey( + "public", + extractable, + [], + algorithm, + handle, + ); + } + case "spki": { + // 1. + if (keyUsages.length > 0) { + throw new DOMException("Invalid key usage", "SyntaxError"); + } + + const publicKeyData = new Uint8Array(56); + if (!op_crypto_import_spki_x448(keyData, publicKeyData)) { + throw new DOMException("Invalid key data", "DataError"); + } + + const handle = {}; + WeakMapPrototypeSet(KEY_STORE, handle, publicKeyData); + + const algorithm = { + name: "X448", + }; + + return constructKey( + "public", + extractable, + [], + algorithm, + handle, + ); + } + case "pkcs8": { + // 1. + if ( + ArrayPrototypeFind( + keyUsages, + (u) => !ArrayPrototypeIncludes(["deriveKey", "deriveBits"], u), + ) !== undefined + ) { + throw new DOMException("Invalid key usage", "SyntaxError"); + } + + const privateKeyData = new Uint8Array(32); + if (!op_crypto_import_pkcs8_x448(keyData, privateKeyData)) { + throw new DOMException("Invalid key data", "DataError"); + } + + const handle = {}; + WeakMapPrototypeSet(KEY_STORE, handle, privateKeyData); + + const algorithm = { + name: "X448", + }; + + return constructKey( + "private", + extractable, + usageIntersection(keyUsages, recognisedUsages), + algorithm, + handle, + ); + } + case "jwk": { + // 1. + const jwk = keyData; + + // 2. + if (jwk.d !== undefined) { + if ( + ArrayPrototypeFind( + keyUsages, + (u) => + !ArrayPrototypeIncludes( + ["deriveKey", "deriveBits"], + u, + ), + ) !== undefined + ) { + throw new DOMException("Invalid key usage", "SyntaxError"); + } + } + + // 3. + if (jwk.d === undefined && keyUsages.length > 0) { + throw new DOMException("Invalid key usage", "SyntaxError"); + } + + // 4. + if (jwk.kty !== "OKP") { + throw new DOMException("Invalid key type", "DataError"); + } + + // 5. + if (jwk.crv !== "X448") { + throw new DOMException("Invalid curve", "DataError"); + } + + // 6. + if (keyUsages.length > 0 && jwk.use !== undefined) { + if (jwk.use !== "enc") { + throw new DOMException("Invalid key use", "DataError"); + } + } + + // 7. + if (jwk.key_ops !== undefined) { + if ( + ArrayPrototypeFind( + jwk.key_ops, + (u) => !ArrayPrototypeIncludes(recognisedUsages, u), + ) !== undefined + ) { + throw new DOMException( + "'key_ops' property of JsonWebKey is invalid", + "DataError", + ); + } + + if ( + !ArrayPrototypeEvery( + jwk.key_ops, + (u) => ArrayPrototypeIncludes(keyUsages, u), + ) + ) { + throw new DOMException( + "'key_ops' property of JsonWebKey is invalid", + "DataError", + ); + } + } + + // 8. + if (jwk.ext !== undefined && jwk.ext === false && extractable) { + throw new DOMException("Invalid key extractability", "DataError"); + } + + // 9. + if (jwk.d !== undefined) { + // https://www.rfc-editor.org/rfc/rfc8037#section-2 + const privateKeyData = op_crypto_base64url_decode(jwk.d); + + const handle = {}; + WeakMapPrototypeSet(KEY_STORE, handle, privateKeyData); + + const algorithm = { + name: "X448", + }; + + return constructKey( + "private", + extractable, + usageIntersection(keyUsages, ["deriveKey", "deriveBits"]), + algorithm, + handle, + ); + } else { + // https://www.rfc-editor.org/rfc/rfc8037#section-2 + const publicKeyData = op_crypto_base64url_decode(jwk.x); + + const handle = {}; + WeakMapPrototypeSet(KEY_STORE, handle, publicKeyData); + + const algorithm = { + name: "X448", + }; + + return constructKey( + "public", + extractable, + [], + algorithm, + handle, + ); + } + } + default: + throw new DOMException("Not implemented", "NotSupportedError"); + } +} + function importKeyEd25519( format, keyData, @@ -3358,6 +3618,14 @@ async function importKeyInner( ["wrapKey", "unwrapKey"], ); } + case "X448": { + return importKeyX448( + format, + keyData, + extractable, + keyUsages, + ); + } case "X25519": { return importKeyX25519( format, @@ -4162,6 +4430,66 @@ function exportKeyEd25519(format, key, innerKey) { } } +function exportKeyX448(format, key, innerKey) { + switch (format) { + case "raw": { + // 1. + if (key[_type] !== "public") { + throw new DOMException( + "Key is not a public key", + "InvalidAccessError", + ); + } + + // 2-3. + return TypedArrayPrototypeGetBuffer(innerKey); + } + case "spki": { + // 1. + if (key[_type] !== "public") { + throw new DOMException( + "Key is not a public key", + "InvalidAccessError", + ); + } + + const spkiDer = op_crypto_export_spki_x448(innerKey); + return TypedArrayPrototypeGetBuffer(spkiDer); + } + case "pkcs8": { + // 1. + if (key[_type] !== "private") { + throw new DOMException( + "Key is not a private key", + "InvalidAccessError", + ); + } + + const pkcs8Der = op_crypto_export_pkcs8_x448( + new Uint8Array([0x04, 0x22, ...new SafeArrayIterator(innerKey)]), + ); + pkcs8Der[15] = 0x20; + return TypedArrayPrototypeGetBuffer(pkcs8Der); + } + case "jwk": { + if (key[_type] === "private") { + throw new DOMException("Not implemented", "NotSupportedError"); + } + const x = op_crypto_base64url_encode(innerKey); + const jwk = { + kty: "OKP", + crv: "X448", + x, + "key_ops": key.usages, + ext: key[_extractable], + }; + return jwk; + } + default: + throw new DOMException("Not implemented", "NotSupportedError"); + } +} + function exportKeyX25519(format, key, innerKey) { switch (format) { case "raw": { @@ -4519,6 +4847,55 @@ async function deriveBits(normalizedAlgorithm, baseKey, length) { return TypedArrayPrototypeGetBuffer(buf); } + case "X448": { + // 1. + if (baseKey[_type] !== "private") { + throw new DOMException("Invalid key type", "InvalidAccessError"); + } + // 2. + const publicKey = normalizedAlgorithm.public; + // 3. + if (publicKey[_type] !== "public") { + throw new DOMException("Invalid key type", "InvalidAccessError"); + } + // 4. + if (publicKey[_algorithm].name !== baseKey[_algorithm].name) { + throw new DOMException( + "Algorithm mismatch", + "InvalidAccessError", + ); + } + + // 5. + const kHandle = baseKey[_handle]; + const k = WeakMapPrototypeGet(KEY_STORE, kHandle); + + const uHandle = publicKey[_handle]; + const u = WeakMapPrototypeGet(KEY_STORE, uHandle); + + const secret = new Uint8Array(56); + const isIdentity = op_crypto_derive_bits_x448(k, u, secret); + + // 6. + if (isIdentity) { + throw new DOMException("Invalid key", "OperationError"); + } + + // 7. + if (length === null) { + return TypedArrayPrototypeGetBuffer(secret); + } else if ( + TypedArrayPrototypeGetByteLength(secret) * 8 < length + ) { + throw new DOMException("Invalid length", "OperationError"); + } else { + return ArrayBufferPrototypeSlice( + TypedArrayPrototypeGetBuffer(secret), + 0, + MathCeil(length / 8), + ); + } + } case "X25519": { // 1. if (baseKey[_type] !== "private") { diff --git a/ext/crypto/Cargo.toml b/ext/crypto/Cargo.toml index 3f2a8d7030359a..d8b740e087e185 100644 --- a/ext/crypto/Cargo.toml +++ b/ext/crypto/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_crypto" -version = "0.182.0" +version = "0.194.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -24,6 +24,7 @@ ctr = "0.9.1" curve25519-dalek = "4.1.3" deno_core.workspace = true deno_web.workspace = true +ed448-goldilocks = { version = "0.8.3", features = ["zeroize"] } elliptic-curve = { version = "0.13.1", features = ["std", "pem"] } num-traits = "0.2.14" once_cell.workspace = true @@ -40,5 +41,7 @@ sha1.workspace = true sha2.workspace = true signature.workspace = true spki.workspace = true +thiserror.workspace = true +tokio.workspace = true uuid.workspace = true x25519-dalek = "2.0.0" diff --git a/ext/crypto/decrypt.rs b/ext/crypto/decrypt.rs index 9b104e1784cb95..1140475183e1cd 100644 --- a/ext/crypto/decrypt.rs +++ b/ext/crypto/decrypt.rs @@ -16,9 +16,6 @@ use ctr::cipher::StreamCipher; use ctr::Ctr128BE; use ctr::Ctr32BE; use ctr::Ctr64BE; -use deno_core::error::custom_error; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::unsync::spawn_blocking; use deno_core::JsBuffer; @@ -73,12 +70,36 @@ pub enum DecryptAlgorithm { }, } +#[derive(Debug, thiserror::Error)] +pub enum DecryptError { + #[error(transparent)] + General(#[from] SharedError), + #[error(transparent)] + Pkcs1(#[from] rsa::pkcs1::Error), + #[error("Decryption failed")] + Failed, + #[error("invalid length")] + InvalidLength, + #[error("invalid counter length. Currently supported 32/64/128 bits")] + InvalidCounterLength, + #[error("tag length not equal to 128")] + InvalidTagLength, + #[error("invalid key or iv")] + InvalidKeyOrIv, + #[error("tried to decrypt too much data")] + TooMuchData, + #[error("iv length not equal to 12 or 16")] + InvalidIvLength, + #[error("{0}")] + Rsa(rsa::Error), +} + #[op2(async)] #[serde] pub async fn op_crypto_decrypt( #[serde] opts: DecryptOptions, #[buffer] data: JsBuffer, -) -> Result { +) -> Result { let key = opts.key; let fun = move || match opts.algorithm { DecryptAlgorithm::RsaOaep { hash, label } => { @@ -108,7 +129,7 @@ fn decrypt_rsa_oaep( hash: ShaHash, label: Vec, data: &[u8], -) -> Result, deno_core::anyhow::Error> { +) -> Result, DecryptError> { let key = key.as_rsa_private_key()?; let private_key = rsa::RsaPrivateKey::from_pkcs1_der(key)?; @@ -139,7 +160,7 @@ fn decrypt_rsa_oaep( private_key .decrypt(padding, data) - .map_err(|e| custom_error("DOMExceptionOperationError", e.to_string())) + .map_err(DecryptError::Rsa) } fn decrypt_aes_cbc( @@ -147,7 +168,7 @@ fn decrypt_aes_cbc( length: usize, iv: Vec, data: &[u8], -) -> Result, deno_core::anyhow::Error> { +) -> Result, DecryptError> { let key = key.as_secret_key()?; // 2. @@ -155,53 +176,32 @@ fn decrypt_aes_cbc( 128 => { // Section 10.3 Step 2 of RFC 2315 https://www.rfc-editor.org/rfc/rfc2315 type Aes128CbcDec = cbc::Decryptor; - let cipher = Aes128CbcDec::new_from_slices(key, &iv).map_err(|_| { - custom_error( - "DOMExceptionOperationError", - "Invalid key or iv".to_string(), - ) - })?; + let cipher = Aes128CbcDec::new_from_slices(key, &iv) + .map_err(|_| DecryptError::InvalidKeyOrIv)?; - cipher.decrypt_padded_vec_mut::(data).map_err(|_| { - custom_error( - "DOMExceptionOperationError", - "Decryption failed".to_string(), - ) - })? + cipher + .decrypt_padded_vec_mut::(data) + .map_err(|_| DecryptError::Failed)? } 192 => { // Section 10.3 Step 2 of RFC 2315 https://www.rfc-editor.org/rfc/rfc2315 type Aes192CbcDec = cbc::Decryptor; - let cipher = Aes192CbcDec::new_from_slices(key, &iv).map_err(|_| { - custom_error( - "DOMExceptionOperationError", - "Invalid key or iv".to_string(), - ) - })?; + let cipher = Aes192CbcDec::new_from_slices(key, &iv) + .map_err(|_| DecryptError::InvalidKeyOrIv)?; - cipher.decrypt_padded_vec_mut::(data).map_err(|_| { - custom_error( - "DOMExceptionOperationError", - "Decryption failed".to_string(), - ) - })? + cipher + .decrypt_padded_vec_mut::(data) + .map_err(|_| DecryptError::Failed)? } 256 => { // Section 10.3 Step 2 of RFC 2315 https://www.rfc-editor.org/rfc/rfc2315 type Aes256CbcDec = cbc::Decryptor; - let cipher = Aes256CbcDec::new_from_slices(key, &iv).map_err(|_| { - custom_error( - "DOMExceptionOperationError", - "Invalid key or iv".to_string(), - ) - })?; + let cipher = Aes256CbcDec::new_from_slices(key, &iv) + .map_err(|_| DecryptError::InvalidKeyOrIv)?; - cipher.decrypt_padded_vec_mut::(data).map_err(|_| { - custom_error( - "DOMExceptionOperationError", - "Decryption failed".to_string(), - ) - })? + cipher + .decrypt_padded_vec_mut::(data) + .map_err(|_| DecryptError::Failed)? } _ => unreachable!(), }; @@ -214,7 +214,7 @@ fn decrypt_aes_ctr_gen( key: &[u8], counter: &[u8], data: &[u8], -) -> Result, AnyError> +) -> Result, DecryptError> where B: KeyIvInit + StreamCipher, { @@ -223,7 +223,7 @@ where let mut plaintext = data.to_vec(); cipher .try_apply_keystream(&mut plaintext) - .map_err(|_| operation_error("tried to decrypt too much data"))?; + .map_err(|_| DecryptError::TooMuchData)?; Ok(plaintext) } @@ -235,12 +235,12 @@ fn decrypt_aes_gcm_gen>( length: usize, additional_data: Vec, plaintext: &mut [u8], -) -> Result<(), AnyError> { +) -> Result<(), DecryptError> { let nonce = Nonce::from_slice(nonce); match length { 128 => { let cipher = aes_gcm::AesGcm::::new_from_slice(key) - .map_err(|_| operation_error("Decryption failed"))?; + .map_err(|_| DecryptError::Failed)?; cipher .decrypt_in_place_detached( nonce, @@ -248,11 +248,11 @@ fn decrypt_aes_gcm_gen>( plaintext, tag, ) - .map_err(|_| operation_error("Decryption failed"))? + .map_err(|_| DecryptError::Failed)? } 192 => { let cipher = aes_gcm::AesGcm::::new_from_slice(key) - .map_err(|_| operation_error("Decryption failed"))?; + .map_err(|_| DecryptError::Failed)?; cipher .decrypt_in_place_detached( nonce, @@ -260,11 +260,11 @@ fn decrypt_aes_gcm_gen>( plaintext, tag, ) - .map_err(|_| operation_error("Decryption failed"))? + .map_err(|_| DecryptError::Failed)? } 256 => { let cipher = aes_gcm::AesGcm::::new_from_slice(key) - .map_err(|_| operation_error("Decryption failed"))?; + .map_err(|_| DecryptError::Failed)?; cipher .decrypt_in_place_detached( nonce, @@ -272,9 +272,9 @@ fn decrypt_aes_gcm_gen>( plaintext, tag, ) - .map_err(|_| operation_error("Decryption failed"))? + .map_err(|_| DecryptError::Failed)? } - _ => return Err(type_error("invalid length")), + _ => return Err(DecryptError::InvalidLength), }; Ok(()) @@ -286,7 +286,7 @@ fn decrypt_aes_ctr( counter: &[u8], ctr_length: usize, data: &[u8], -) -> Result, deno_core::anyhow::Error> { +) -> Result, DecryptError> { let key = key.as_secret_key()?; match ctr_length { @@ -294,23 +294,21 @@ fn decrypt_aes_ctr( 128 => decrypt_aes_ctr_gen::>(key, counter, data), 192 => decrypt_aes_ctr_gen::>(key, counter, data), 256 => decrypt_aes_ctr_gen::>(key, counter, data), - _ => Err(type_error("invalid length")), + _ => Err(DecryptError::InvalidLength), }, 64 => match key_length { 128 => decrypt_aes_ctr_gen::>(key, counter, data), 192 => decrypt_aes_ctr_gen::>(key, counter, data), 256 => decrypt_aes_ctr_gen::>(key, counter, data), - _ => Err(type_error("invalid length")), + _ => Err(DecryptError::InvalidLength), }, 128 => match key_length { 128 => decrypt_aes_ctr_gen::>(key, counter, data), 192 => decrypt_aes_ctr_gen::>(key, counter, data), 256 => decrypt_aes_ctr_gen::>(key, counter, data), - _ => Err(type_error("invalid length")), + _ => Err(DecryptError::InvalidLength), }, - _ => Err(type_error( - "invalid counter length. Currently supported 32/64/128 bits", - )), + _ => Err(DecryptError::InvalidCounterLength), } } @@ -321,7 +319,7 @@ fn decrypt_aes_gcm( iv: Vec, additional_data: Option>, data: &[u8], -) -> Result, AnyError> { +) -> Result, DecryptError> { let key = key.as_secret_key()?; let additional_data = additional_data.unwrap_or_default(); @@ -330,7 +328,7 @@ fn decrypt_aes_gcm( // Note that encryption won't fail, it instead truncates the tag // to the specified tag length as specified in the spec. if tag_length != 128 { - return Err(type_error("tag length not equal to 128")); + return Err(DecryptError::InvalidTagLength); } let sep = data.len() - (tag_length / 8); @@ -357,7 +355,7 @@ fn decrypt_aes_gcm( additional_data, &mut plaintext, )?, - _ => return Err(type_error("iv length not equal to 12 or 16")), + _ => return Err(DecryptError::InvalidIvLength), } Ok(plaintext) diff --git a/ext/crypto/ed25519.rs b/ext/crypto/ed25519.rs index 4f604fe5135f08..da34b7d25dbf05 100644 --- a/ext/crypto/ed25519.rs +++ b/ext/crypto/ed25519.rs @@ -2,8 +2,6 @@ use base64::prelude::BASE64_URL_SAFE_NO_PAD; use base64::Engine; -use deno_core::error::custom_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::ToJsBuffer; use elliptic_curve::pkcs8::PrivateKeyInfo; @@ -15,6 +13,16 @@ use spki::der::asn1::BitString; use spki::der::Decode; use spki::der::Encode; +#[derive(Debug, thiserror::Error)] +pub enum Ed25519Error { + #[error("Failed to export key")] + FailedExport, + #[error(transparent)] + Der(#[from] rsa::pkcs1::der::Error), + #[error(transparent)] + KeyRejected(#[from] ring::error::KeyRejected), +} + #[op2(fast)] pub fn op_crypto_generate_ed25519_keypair( #[buffer] pkey: &mut [u8], @@ -116,7 +124,7 @@ pub fn op_crypto_import_pkcs8_ed25519( #[serde] pub fn op_crypto_export_spki_ed25519( #[buffer] pubkey: &[u8], -) -> Result { +) -> Result { let key_info = spki::SubjectPublicKeyInfo { algorithm: spki::AlgorithmIdentifierOwned { // id-Ed25519 @@ -128,9 +136,7 @@ pub fn op_crypto_export_spki_ed25519( Ok( key_info .to_der() - .map_err(|_| { - custom_error("DOMExceptionOperationError", "Failed to export key") - })? + .map_err(|_| Ed25519Error::FailedExport)? .into(), ) } @@ -139,7 +145,7 @@ pub fn op_crypto_export_spki_ed25519( #[serde] pub fn op_crypto_export_pkcs8_ed25519( #[buffer] pkey: &[u8], -) -> Result { +) -> Result { use rsa::pkcs1::der::Encode; // This should probably use OneAsymmetricKey instead @@ -164,7 +170,7 @@ pub fn op_crypto_export_pkcs8_ed25519( #[string] pub fn op_crypto_jwk_x_ed25519( #[buffer] pkey: &[u8], -) -> Result { +) -> Result { let pair = Ed25519KeyPair::from_seed_unchecked(pkey)?; Ok(BASE64_URL_SAFE_NO_PAD.encode(pair.public_key().as_ref())) } diff --git a/ext/crypto/encrypt.rs b/ext/crypto/encrypt.rs index 204648e892fe55..66b27657f8f055 100644 --- a/ext/crypto/encrypt.rs +++ b/ext/crypto/encrypt.rs @@ -16,8 +16,6 @@ use aes_gcm::Nonce; use ctr::Ctr128BE; use ctr::Ctr32BE; use ctr::Ctr64BE; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::unsync::spawn_blocking; use deno_core::JsBuffer; @@ -73,12 +71,30 @@ pub enum EncryptAlgorithm { }, } +#[derive(Debug, thiserror::Error)] +pub enum EncryptError { + #[error(transparent)] + General(#[from] SharedError), + #[error("invalid length")] + InvalidLength, + #[error("invalid key or iv")] + InvalidKeyOrIv, + #[error("iv length not equal to 12 or 16")] + InvalidIvLength, + #[error("invalid counter length. Currently supported 32/64/128 bits")] + InvalidCounterLength, + #[error("tried to encrypt too much data")] + TooMuchData, + #[error("Encryption failed")] + Failed, +} + #[op2(async)] #[serde] pub async fn op_crypto_encrypt( #[serde] opts: EncryptOptions, #[buffer] data: JsBuffer, -) -> Result { +) -> Result { let key = opts.key; let fun = move || match opts.algorithm { EncryptAlgorithm::RsaOaep { hash, label } => { @@ -108,12 +124,12 @@ fn encrypt_rsa_oaep( hash: ShaHash, label: Vec, data: &[u8], -) -> Result, AnyError> { +) -> Result, EncryptError> { let label = String::from_utf8_lossy(&label).to_string(); let public_key = key.as_rsa_public_key()?; let public_key = rsa::RsaPublicKey::from_pkcs1_der(&public_key) - .map_err(|_| operation_error("failed to decode public key"))?; + .map_err(|_| SharedError::FailedDecodePublicKey)?; let mut rng = OsRng; let padding = match hash { ShaHash::Sha1 => rsa::Oaep { @@ -139,7 +155,7 @@ fn encrypt_rsa_oaep( }; let encrypted = public_key .encrypt(&mut rng, padding, data) - .map_err(|_| operation_error("Encryption failed"))?; + .map_err(|_| EncryptError::Failed)?; Ok(encrypted) } @@ -148,7 +164,7 @@ fn encrypt_aes_cbc( length: usize, iv: Vec, data: &[u8], -) -> Result, AnyError> { +) -> Result, EncryptError> { let key = key.as_secret_key()?; let ciphertext = match length { 128 => { @@ -156,7 +172,7 @@ fn encrypt_aes_cbc( type Aes128CbcEnc = cbc::Encryptor; let cipher = Aes128CbcEnc::new_from_slices(key, &iv) - .map_err(|_| operation_error("invalid key or iv".to_string()))?; + .map_err(|_| EncryptError::InvalidKeyOrIv)?; cipher.encrypt_padded_vec_mut::(data) } 192 => { @@ -164,7 +180,7 @@ fn encrypt_aes_cbc( type Aes192CbcEnc = cbc::Encryptor; let cipher = Aes192CbcEnc::new_from_slices(key, &iv) - .map_err(|_| operation_error("invalid key or iv".to_string()))?; + .map_err(|_| EncryptError::InvalidKeyOrIv)?; cipher.encrypt_padded_vec_mut::(data) } 256 => { @@ -172,10 +188,10 @@ fn encrypt_aes_cbc( type Aes256CbcEnc = cbc::Encryptor; let cipher = Aes256CbcEnc::new_from_slices(key, &iv) - .map_err(|_| operation_error("invalid key or iv".to_string()))?; + .map_err(|_| EncryptError::InvalidKeyOrIv)?; cipher.encrypt_padded_vec_mut::(data) } - _ => return Err(type_error("invalid length")), + _ => return Err(EncryptError::InvalidLength), }; Ok(ciphertext) } @@ -186,31 +202,31 @@ fn encrypt_aes_gcm_general>( length: usize, ciphertext: &mut [u8], additional_data: Vec, -) -> Result { +) -> Result { let nonce = Nonce::::from_slice(&iv); let tag = match length { 128 => { let cipher = aes_gcm::AesGcm::::new_from_slice(key) - .map_err(|_| operation_error("Encryption failed"))?; + .map_err(|_| EncryptError::Failed)?; cipher .encrypt_in_place_detached(nonce, &additional_data, ciphertext) - .map_err(|_| operation_error("Encryption failed"))? + .map_err(|_| EncryptError::Failed)? } 192 => { let cipher = aes_gcm::AesGcm::::new_from_slice(key) - .map_err(|_| operation_error("Encryption failed"))?; + .map_err(|_| EncryptError::Failed)?; cipher .encrypt_in_place_detached(nonce, &additional_data, ciphertext) - .map_err(|_| operation_error("Encryption failed"))? + .map_err(|_| EncryptError::Failed)? } 256 => { let cipher = aes_gcm::AesGcm::::new_from_slice(key) - .map_err(|_| operation_error("Encryption failed"))?; + .map_err(|_| EncryptError::Failed)?; cipher .encrypt_in_place_detached(nonce, &additional_data, ciphertext) - .map_err(|_| operation_error("Encryption failed"))? + .map_err(|_| EncryptError::Failed)? } - _ => return Err(type_error("invalid length")), + _ => return Err(EncryptError::InvalidLength), }; Ok(tag) @@ -223,7 +239,7 @@ fn encrypt_aes_gcm( iv: Vec, additional_data: Option>, data: &[u8], -) -> Result, AnyError> { +) -> Result, EncryptError> { let key = key.as_secret_key()?; let additional_data = additional_data.unwrap_or_default(); @@ -244,7 +260,7 @@ fn encrypt_aes_gcm( &mut ciphertext, additional_data, )?, - _ => return Err(type_error("iv length not equal to 12 or 16")), + _ => return Err(EncryptError::InvalidIvLength), }; // Truncated tag to the specified tag length. @@ -261,7 +277,7 @@ fn encrypt_aes_ctr_gen( key: &[u8], counter: &[u8], data: &[u8], -) -> Result, AnyError> +) -> Result, EncryptError> where B: KeyIvInit + StreamCipher, { @@ -270,7 +286,7 @@ where let mut ciphertext = data.to_vec(); cipher .try_apply_keystream(&mut ciphertext) - .map_err(|_| operation_error("tried to encrypt too much data"))?; + .map_err(|_| EncryptError::TooMuchData)?; Ok(ciphertext) } @@ -281,7 +297,7 @@ fn encrypt_aes_ctr( counter: &[u8], ctr_length: usize, data: &[u8], -) -> Result, AnyError> { +) -> Result, EncryptError> { let key = key.as_secret_key()?; match ctr_length { @@ -289,22 +305,20 @@ fn encrypt_aes_ctr( 128 => encrypt_aes_ctr_gen::>(key, counter, data), 192 => encrypt_aes_ctr_gen::>(key, counter, data), 256 => encrypt_aes_ctr_gen::>(key, counter, data), - _ => Err(type_error("invalid length")), + _ => Err(EncryptError::InvalidLength), }, 64 => match key_length { 128 => encrypt_aes_ctr_gen::>(key, counter, data), 192 => encrypt_aes_ctr_gen::>(key, counter, data), 256 => encrypt_aes_ctr_gen::>(key, counter, data), - _ => Err(type_error("invalid length")), + _ => Err(EncryptError::InvalidLength), }, 128 => match key_length { 128 => encrypt_aes_ctr_gen::>(key, counter, data), 192 => encrypt_aes_ctr_gen::>(key, counter, data), 256 => encrypt_aes_ctr_gen::>(key, counter, data), - _ => Err(type_error("invalid length")), + _ => Err(EncryptError::InvalidLength), }, - _ => Err(type_error( - "invalid counter length. Currently supported 32/64/128 bits", - )), + _ => Err(EncryptError::InvalidCounterLength), } } diff --git a/ext/crypto/export_key.rs b/ext/crypto/export_key.rs index 00ce7e11c66b6b..edf0d7239c70bb 100644 --- a/ext/crypto/export_key.rs +++ b/ext/crypto/export_key.rs @@ -4,8 +4,6 @@ use base64::prelude::BASE64_URL_SAFE_NO_PAD; use base64::Engine; use const_oid::AssociatedOid; use const_oid::ObjectIdentifier; -use deno_core::error::custom_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::ToJsBuffer; use elliptic_curve::sec1::ToEncodedPoint; @@ -22,6 +20,16 @@ use spki::AlgorithmIdentifierOwned; use crate::shared::*; +#[derive(Debug, thiserror::Error)] +pub enum ExportKeyError { + #[error(transparent)] + General(#[from] SharedError), + #[error(transparent)] + Der(#[from] spki::der::Error), + #[error("Unsupported named curve")] + UnsupportedNamedCurve, +} + #[derive(Deserialize)] #[serde(rename_all = "camelCase")] pub struct ExportKeyOptions { @@ -99,7 +107,7 @@ pub enum ExportKeyResult { pub fn op_crypto_export_key( #[serde] opts: ExportKeyOptions, #[serde] key_data: V8RawKeyData, -) -> Result { +) -> Result { match opts.algorithm { ExportKeyAlgorithm::RsassaPkcs1v15 {} | ExportKeyAlgorithm::RsaPss {} @@ -125,7 +133,7 @@ fn bytes_to_b64(bytes: &[u8]) -> String { fn export_key_rsa( format: ExportKeyFormat, key_data: V8RawKeyData, -) -> Result { +) -> Result { match format { ExportKeyFormat::Spki => { let subject_public_key = &key_data.as_rsa_public_key()?; @@ -181,12 +189,7 @@ fn export_key_rsa( ExportKeyFormat::JwkPublic => { let public_key = key_data.as_rsa_public_key()?; let public_key = rsa::pkcs1::RsaPublicKey::from_der(&public_key) - .map_err(|_| { - custom_error( - "DOMExceptionOperationError", - "failed to decode public key", - ) - })?; + .map_err(|_| SharedError::FailedDecodePublicKey)?; Ok(ExportKeyResult::JwkPublicRsa { n: uint_to_b64(public_key.modulus), @@ -196,12 +199,7 @@ fn export_key_rsa( ExportKeyFormat::JwkPrivate => { let private_key = key_data.as_rsa_private_key()?; let private_key = rsa::pkcs1::RsaPrivateKey::from_der(private_key) - .map_err(|_| { - custom_error( - "DOMExceptionOperationError", - "failed to decode private key", - ) - })?; + .map_err(|_| SharedError::FailedDecodePrivateKey)?; Ok(ExportKeyResult::JwkPrivateRsa { n: uint_to_b64(private_key.modulus), @@ -214,14 +212,14 @@ fn export_key_rsa( qi: uint_to_b64(private_key.coefficient), }) } - _ => Err(unsupported_format()), + _ => Err(SharedError::UnsupportedFormat.into()), } } fn export_key_symmetric( format: ExportKeyFormat, key_data: V8RawKeyData, -) -> Result { +) -> Result { match format { ExportKeyFormat::JwkSecret => { let bytes = key_data.as_secret_key()?; @@ -230,7 +228,7 @@ fn export_key_symmetric( k: bytes_to_b64(bytes), }) } - _ => Err(unsupported_format()), + _ => Err(SharedError::UnsupportedFormat.into()), } } @@ -239,7 +237,7 @@ fn export_key_ec( key_data: V8RawKeyData, algorithm: ExportKeyAlgorithm, named_curve: EcNamedCurve, -) -> Result { +) -> Result { match format { ExportKeyFormat::Raw => { let subject_public_key = match named_curve { @@ -332,10 +330,7 @@ fn export_key_ec( y: bytes_to_b64(y), }) } else { - Err(custom_error( - "DOMExceptionOperationError", - "failed to decode public key", - )) + Err(SharedError::FailedDecodePublicKey.into()) } } EcNamedCurve::P384 => { @@ -350,10 +345,7 @@ fn export_key_ec( y: bytes_to_b64(y), }) } else { - Err(custom_error( - "DOMExceptionOperationError", - "failed to decode public key", - )) + Err(SharedError::FailedDecodePublicKey.into()) } } EcNamedCurve::P521 => { @@ -368,10 +360,7 @@ fn export_key_ec( y: bytes_to_b64(y), }) } else { - Err(custom_error( - "DOMExceptionOperationError", - "failed to decode public key", - )) + Err(SharedError::FailedDecodePublicKey.into()) } } }, @@ -380,13 +369,8 @@ fn export_key_ec( match named_curve { EcNamedCurve::P256 => { - let ec_key = - p256::SecretKey::from_pkcs8_der(private_key).map_err(|_| { - custom_error( - "DOMExceptionOperationError", - "failed to decode private key", - ) - })?; + let ec_key = p256::SecretKey::from_pkcs8_der(private_key) + .map_err(|_| SharedError::FailedDecodePrivateKey)?; let point = ec_key.public_key().to_encoded_point(false); if let elliptic_curve::sec1::Coordinates::Uncompressed { x, y } = @@ -398,18 +382,13 @@ fn export_key_ec( d: bytes_to_b64(&ec_key.to_bytes()), }) } else { - Err(data_error("expected valid public EC key")) + Err(SharedError::ExpectedValidPublicECKey.into()) } } EcNamedCurve::P384 => { - let ec_key = - p384::SecretKey::from_pkcs8_der(private_key).map_err(|_| { - custom_error( - "DOMExceptionOperationError", - "failed to decode private key", - ) - })?; + let ec_key = p384::SecretKey::from_pkcs8_der(private_key) + .map_err(|_| SharedError::FailedDecodePrivateKey)?; let point = ec_key.public_key().to_encoded_point(false); if let elliptic_curve::sec1::Coordinates::Uncompressed { x, y } = @@ -421,12 +400,12 @@ fn export_key_ec( d: bytes_to_b64(&ec_key.to_bytes()), }) } else { - Err(data_error("expected valid public EC key")) + Err(SharedError::ExpectedValidPublicECKey.into()) } } - _ => Err(not_supported_error("Unsupported namedCurve")), + _ => Err(ExportKeyError::UnsupportedNamedCurve), } } - ExportKeyFormat::JwkSecret => Err(unsupported_format()), + ExportKeyFormat::JwkSecret => Err(SharedError::UnsupportedFormat.into()), } } diff --git a/ext/crypto/generate_key.rs b/ext/crypto/generate_key.rs index 43aea2c705c214..3c0bd77c22641c 100644 --- a/ext/crypto/generate_key.rs +++ b/ext/crypto/generate_key.rs @@ -1,6 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::AnyError; use deno_core::op2; use deno_core::unsync::spawn_blocking; use deno_core::ToJsBuffer; @@ -16,6 +15,26 @@ use serde::Deserialize; use crate::shared::*; +#[derive(Debug, thiserror::Error)] +pub enum GenerateKeyError { + #[error(transparent)] + General(#[from] SharedError), + #[error("Bad public exponent")] + BadPublicExponent, + #[error("Invalid HMAC key length")] + InvalidHMACKeyLength, + #[error("Failed to serialize RSA key")] + FailedRSAKeySerialization, + #[error("Invalid AES key length")] + InvalidAESKeyLength, + #[error("Failed to generate RSA key")] + FailedRSAKeyGeneration, + #[error("Failed to generate EC key")] + FailedECKeyGeneration, + #[error("Failed to generate key")] + FailedKeyGeneration, +} + // Allowlist for RSA public exponents. static PUB_EXPONENT_1: Lazy = Lazy::new(|| BigUint::from_u64(3).unwrap()); @@ -46,7 +65,7 @@ pub enum GenerateKeyOptions { #[serde] pub async fn op_crypto_generate_key( #[serde] opts: GenerateKeyOptions, -) -> Result { +) -> Result { let fun = || match opts { GenerateKeyOptions::Rsa { modulus_length, @@ -65,21 +84,21 @@ pub async fn op_crypto_generate_key( fn generate_key_rsa( modulus_length: u32, public_exponent: &[u8], -) -> Result, AnyError> { +) -> Result, GenerateKeyError> { let exponent = BigUint::from_bytes_be(public_exponent); if exponent != *PUB_EXPONENT_1 && exponent != *PUB_EXPONENT_2 { - return Err(operation_error("Bad public exponent")); + return Err(GenerateKeyError::BadPublicExponent); } let mut rng = OsRng; let private_key = RsaPrivateKey::new_with_exp(&mut rng, modulus_length as usize, &exponent) - .map_err(|_| operation_error("Failed to generate RSA key"))?; + .map_err(|_| GenerateKeyError::FailedRSAKeyGeneration)?; let private_key = private_key .to_pkcs1_der() - .map_err(|_| operation_error("Failed to serialize RSA key"))?; + .map_err(|_| GenerateKeyError::FailedRSAKeySerialization)?; Ok(private_key.as_bytes().to_vec()) } @@ -90,7 +109,9 @@ fn generate_key_ec_p521() -> Vec { key.to_nonzero_scalar().to_bytes().to_vec() } -fn generate_key_ec(named_curve: EcNamedCurve) -> Result, AnyError> { +fn generate_key_ec( + named_curve: EcNamedCurve, +) -> Result, GenerateKeyError> { let curve = match named_curve { EcNamedCurve::P256 => &ring::signature::ECDSA_P256_SHA256_FIXED_SIGNING, EcNamedCurve::P384 => &ring::signature::ECDSA_P384_SHA384_FIXED_SIGNING, @@ -100,21 +121,21 @@ fn generate_key_ec(named_curve: EcNamedCurve) -> Result, AnyError> { let rng = ring::rand::SystemRandom::new(); let pkcs8 = EcdsaKeyPair::generate_pkcs8(curve, &rng) - .map_err(|_| operation_error("Failed to generate EC key"))?; + .map_err(|_| GenerateKeyError::FailedECKeyGeneration)?; Ok(pkcs8.as_ref().to_vec()) } -fn generate_key_aes(length: usize) -> Result, AnyError> { +fn generate_key_aes(length: usize) -> Result, GenerateKeyError> { if length % 8 != 0 || length > 256 { - return Err(operation_error("Invalid AES key length")); + return Err(GenerateKeyError::InvalidAESKeyLength); } let mut key = vec![0u8; length / 8]; let rng = ring::rand::SystemRandom::new(); rng .fill(&mut key) - .map_err(|_| operation_error("Failed to generate key"))?; + .map_err(|_| GenerateKeyError::FailedKeyGeneration)?; Ok(key) } @@ -122,7 +143,7 @@ fn generate_key_aes(length: usize) -> Result, AnyError> { fn generate_key_hmac( hash: ShaHash, length: Option, -) -> Result, AnyError> { +) -> Result, GenerateKeyError> { let hash = match hash { ShaHash::Sha1 => &ring::hmac::HMAC_SHA1_FOR_LEGACY_USE_ONLY, ShaHash::Sha256 => &ring::hmac::HMAC_SHA256, @@ -132,12 +153,12 @@ fn generate_key_hmac( let length = if let Some(length) = length { if length % 8 != 0 { - return Err(operation_error("Invalid HMAC key length")); + return Err(GenerateKeyError::InvalidHMACKeyLength); } let length = length / 8; if length > ring::digest::MAX_BLOCK_LEN { - return Err(operation_error("Invalid HMAC key length")); + return Err(GenerateKeyError::InvalidHMACKeyLength); } length @@ -149,7 +170,7 @@ fn generate_key_hmac( let mut key = vec![0u8; length]; rng .fill(&mut key) - .map_err(|_| operation_error("Failed to generate key"))?; + .map_err(|_| GenerateKeyError::FailedKeyGeneration)?; Ok(key) } diff --git a/ext/crypto/import_key.rs b/ext/crypto/import_key.rs index e30baea03ae7fc..3463ca2beb34e9 100644 --- a/ext/crypto/import_key.rs +++ b/ext/crypto/import_key.rs @@ -1,7 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use base64::Engine; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::JsBuffer; use deno_core::ToJsBuffer; @@ -15,6 +14,70 @@ use spki::der::Decode; use crate::shared::*; +#[derive(Debug, thiserror::Error)] +pub enum ImportKeyError { + #[error(transparent)] + General(#[from] SharedError), + #[error("invalid modulus")] + InvalidModulus, + #[error("invalid public exponent")] + InvalidPublicExponent, + #[error("invalid private exponent")] + InvalidPrivateExponent, + #[error("invalid first prime factor")] + InvalidFirstPrimeFactor, + #[error("invalid second prime factor")] + InvalidSecondPrimeFactor, + #[error("invalid first CRT exponent")] + InvalidFirstCRTExponent, + #[error("invalid second CRT exponent")] + InvalidSecondCRTExponent, + #[error("invalid CRT coefficient")] + InvalidCRTCoefficient, + #[error("invalid b64 coordinate")] + InvalidB64Coordinate, + #[error("invalid RSA public key")] + InvalidRSAPublicKey, + #[error("invalid RSA private key")] + InvalidRSAPrivateKey, + #[error("unsupported algorithm")] + UnsupportedAlgorithm, + #[error("public key is invalid (too long)")] + PublicKeyTooLong, + #[error("private key is invalid (too long)")] + PrivateKeyTooLong, + #[error("invalid P-256 elliptic curve point")] + InvalidP256ECPoint, + #[error("invalid P-384 elliptic curve point")] + InvalidP384ECPoint, + #[error("invalid P-521 elliptic curve point")] + InvalidP521ECPoint, + #[error("invalid P-256 elliptic curve SPKI data")] + InvalidP256ECSPKIData, + #[error("invalid P-384 elliptic curve SPKI data")] + InvalidP384ECSPKIData, + #[error("invalid P-521 elliptic curve SPKI data")] + InvalidP521ECSPKIData, + #[error("curve mismatch")] + CurveMismatch, + #[error("Unsupported named curve")] + UnsupportedNamedCurve, + #[error("invalid key data")] + InvalidKeyData, + #[error("invalid JWK private key")] + InvalidJWKPrivateKey, + #[error(transparent)] + EllipticCurve(#[from] elliptic_curve::Error), + #[error("expected valid PKCS#8 data")] + ExpectedValidPkcs8Data, + #[error("malformed parameters")] + MalformedParameters, + #[error(transparent)] + Spki(#[from] spki::Error), + #[error(transparent)] + Der(#[from] rsa::pkcs1::der::Error), +} + #[derive(Deserialize)] #[serde(rename_all = "camelCase")] pub enum KeyData { @@ -93,7 +156,7 @@ pub enum ImportKeyResult { pub fn op_crypto_import_key( #[serde] opts: ImportKeyOptions, #[serde] key_data: KeyData, -) -> Result { +) -> Result { match opts { ImportKeyOptions::RsassaPkcs1v15 {} => import_key_rsassa(key_data), ImportKeyOptions::RsaPss {} => import_key_rsapss(key_data), @@ -117,21 +180,21 @@ const BASE64_URL_SAFE_FORGIVING: ); macro_rules! jwt_b64_int_or_err { - ($name:ident, $b64:expr, $err:expr) => { + ($name:ident, $b64:expr, $err:tt) => { let bytes = BASE64_URL_SAFE_FORGIVING .decode($b64) - .map_err(|_| data_error($err))?; - let $name = UintRef::new(&bytes).map_err(|_| data_error($err))?; + .map_err(|_| ImportKeyError::$err)?; + let $name = UintRef::new(&bytes).map_err(|_| ImportKeyError::$err)?; }; } fn import_key_rsa_jwk( key_data: KeyData, -) -> Result { +) -> Result { match key_data { KeyData::JwkPublicRsa { n, e } => { - jwt_b64_int_or_err!(modulus, &n, "invalid modulus"); - jwt_b64_int_or_err!(public_exponent, &e, "invalid public exponent"); + jwt_b64_int_or_err!(modulus, &n, InvalidModulus); + jwt_b64_int_or_err!(public_exponent, &e, InvalidPublicExponent); let public_key = rsa::pkcs1::RsaPublicKey { modulus, @@ -141,7 +204,7 @@ fn import_key_rsa_jwk( let mut data = Vec::new(); public_key .encode_to_vec(&mut data) - .map_err(|_| data_error("invalid rsa public key"))?; + .map_err(|_| ImportKeyError::InvalidRSAPublicKey)?; let public_exponent = public_key.public_exponent.as_bytes().to_vec().into(); @@ -163,14 +226,14 @@ fn import_key_rsa_jwk( dq, qi, } => { - jwt_b64_int_or_err!(modulus, &n, "invalid modulus"); - jwt_b64_int_or_err!(public_exponent, &e, "invalid public exponent"); - jwt_b64_int_or_err!(private_exponent, &d, "invalid private exponent"); - jwt_b64_int_or_err!(prime1, &p, "invalid first prime factor"); - jwt_b64_int_or_err!(prime2, &q, "invalid second prime factor"); - jwt_b64_int_or_err!(exponent1, &dp, "invalid first CRT exponent"); - jwt_b64_int_or_err!(exponent2, &dq, "invalid second CRT exponent"); - jwt_b64_int_or_err!(coefficient, &qi, "invalid CRT coefficient"); + jwt_b64_int_or_err!(modulus, &n, InvalidModulus); + jwt_b64_int_or_err!(public_exponent, &e, InvalidPublicExponent); + jwt_b64_int_or_err!(private_exponent, &d, InvalidPrivateExponent); + jwt_b64_int_or_err!(prime1, &p, InvalidFirstPrimeFactor); + jwt_b64_int_or_err!(prime2, &q, InvalidSecondPrimeFactor); + jwt_b64_int_or_err!(exponent1, &dp, InvalidFirstCRTExponent); + jwt_b64_int_or_err!(exponent2, &dq, InvalidSecondCRTExponent); + jwt_b64_int_or_err!(coefficient, &qi, InvalidCRTCoefficient); let private_key = rsa::pkcs1::RsaPrivateKey { modulus, @@ -187,7 +250,7 @@ fn import_key_rsa_jwk( let mut data = Vec::new(); private_key .encode_to_vec(&mut data) - .map_err(|_| data_error("invalid rsa private key"))?; + .map_err(|_| ImportKeyError::InvalidRSAPrivateKey)?; let public_exponent = private_key.public_exponent.as_bytes().to_vec().into(); @@ -205,37 +268,33 @@ fn import_key_rsa_jwk( fn import_key_rsassa( key_data: KeyData, -) -> Result { +) -> Result { match key_data { KeyData::Spki(data) => { // 2-3. - let pk_info = spki::SubjectPublicKeyInfoRef::try_from(&*data) - .map_err(|e| data_error(e.to_string()))?; + let pk_info = spki::SubjectPublicKeyInfoRef::try_from(&*data)?; // 4-5. let alg = pk_info.algorithm.oid; // 6-7. (skipped, only support rsaEncryption for interoperability) if alg != RSA_ENCRYPTION_OID { - return Err(data_error("unsupported algorithm")); + return Err(ImportKeyError::UnsupportedAlgorithm); } // 8-9. let public_key = rsa::pkcs1::RsaPublicKey::from_der( pk_info.subject_public_key.raw_bytes(), - ) - .map_err(|e| data_error(e.to_string()))?; + )?; - let bytes_consumed = public_key - .encoded_len() - .map_err(|e| data_error(e.to_string()))?; + let bytes_consumed = public_key.encoded_len()?; if bytes_consumed != rsa::pkcs1::der::Length::new( pk_info.subject_public_key.raw_bytes().len() as u16, ) { - return Err(data_error("public key is invalid (too long)")); + return Err(ImportKeyError::PublicKeyTooLong); } let data = pk_info.subject_public_key.raw_bytes().to_vec().into(); @@ -251,30 +310,26 @@ fn import_key_rsassa( } KeyData::Pkcs8(data) => { // 2-3. - let pk_info = PrivateKeyInfo::from_der(&data) - .map_err(|e| data_error(e.to_string()))?; + let pk_info = PrivateKeyInfo::from_der(&data)?; // 4-5. let alg = pk_info.algorithm.oid; // 6-7. (skipped, only support rsaEncryption for interoperability) if alg != RSA_ENCRYPTION_OID { - return Err(data_error("unsupported algorithm")); + return Err(ImportKeyError::UnsupportedAlgorithm); } // 8-9. let private_key = - rsa::pkcs1::RsaPrivateKey::from_der(pk_info.private_key) - .map_err(|e| data_error(e.to_string()))?; + rsa::pkcs1::RsaPrivateKey::from_der(pk_info.private_key)?; - let bytes_consumed = private_key - .encoded_len() - .map_err(|e| data_error(e.to_string()))?; + let bytes_consumed = private_key.encoded_len()?; if bytes_consumed != rsa::pkcs1::der::Length::new(pk_info.private_key.len() as u16) { - return Err(data_error("private key is invalid (too long)")); + return Err(ImportKeyError::PrivateKeyTooLong); } let data = pk_info.private_key.to_vec().into(); @@ -291,43 +346,39 @@ fn import_key_rsassa( KeyData::JwkPublicRsa { .. } | KeyData::JwkPrivateRsa { .. } => { import_key_rsa_jwk(key_data) } - _ => Err(unsupported_format()), + _ => Err(SharedError::UnsupportedFormat.into()), } } fn import_key_rsapss( key_data: KeyData, -) -> Result { +) -> Result { match key_data { KeyData::Spki(data) => { // 2-3. - let pk_info = spki::SubjectPublicKeyInfoRef::try_from(&*data) - .map_err(|e| data_error(e.to_string()))?; + let pk_info = spki::SubjectPublicKeyInfoRef::try_from(&*data)?; // 4-5. let alg = pk_info.algorithm.oid; // 6-7. (skipped, only support rsaEncryption for interoperability) if alg != RSA_ENCRYPTION_OID { - return Err(data_error("unsupported algorithm")); + return Err(ImportKeyError::UnsupportedAlgorithm); } // 8-9. let public_key = rsa::pkcs1::RsaPublicKey::from_der( pk_info.subject_public_key.raw_bytes(), - ) - .map_err(|e| data_error(e.to_string()))?; + )?; - let bytes_consumed = public_key - .encoded_len() - .map_err(|e| data_error(e.to_string()))?; + let bytes_consumed = public_key.encoded_len()?; if bytes_consumed != rsa::pkcs1::der::Length::new( pk_info.subject_public_key.raw_bytes().len() as u16, ) { - return Err(data_error("public key is invalid (too long)")); + return Err(ImportKeyError::PublicKeyTooLong); } let data = pk_info.subject_public_key.raw_bytes().to_vec().into(); @@ -343,30 +394,26 @@ fn import_key_rsapss( } KeyData::Pkcs8(data) => { // 2-3. - let pk_info = PrivateKeyInfo::from_der(&data) - .map_err(|e| data_error(e.to_string()))?; + let pk_info = PrivateKeyInfo::from_der(&data)?; // 4-5. let alg = pk_info.algorithm.oid; // 6-7. (skipped, only support rsaEncryption for interoperability) if alg != RSA_ENCRYPTION_OID { - return Err(data_error("unsupported algorithm")); + return Err(ImportKeyError::UnsupportedAlgorithm); } // 8-9. let private_key = - rsa::pkcs1::RsaPrivateKey::from_der(pk_info.private_key) - .map_err(|e| data_error(e.to_string()))?; + rsa::pkcs1::RsaPrivateKey::from_der(pk_info.private_key)?; - let bytes_consumed = private_key - .encoded_len() - .map_err(|e| data_error(e.to_string()))?; + let bytes_consumed = private_key.encoded_len()?; if bytes_consumed != rsa::pkcs1::der::Length::new(pk_info.private_key.len() as u16) { - return Err(data_error("private key is invalid (too long)")); + return Err(ImportKeyError::PrivateKeyTooLong); } let data = pk_info.private_key.to_vec().into(); @@ -383,43 +430,39 @@ fn import_key_rsapss( KeyData::JwkPublicRsa { .. } | KeyData::JwkPrivateRsa { .. } => { import_key_rsa_jwk(key_data) } - _ => Err(unsupported_format()), + _ => Err(SharedError::UnsupportedFormat.into()), } } fn import_key_rsaoaep( key_data: KeyData, -) -> Result { +) -> Result { match key_data { KeyData::Spki(data) => { // 2-3. - let pk_info = spki::SubjectPublicKeyInfoRef::try_from(&*data) - .map_err(|e| data_error(e.to_string()))?; + let pk_info = spki::SubjectPublicKeyInfoRef::try_from(&*data)?; // 4-5. let alg = pk_info.algorithm.oid; // 6-7. (skipped, only support rsaEncryption for interoperability) if alg != RSA_ENCRYPTION_OID { - return Err(data_error("unsupported algorithm")); + return Err(ImportKeyError::UnsupportedAlgorithm); } // 8-9. let public_key = rsa::pkcs1::RsaPublicKey::from_der( pk_info.subject_public_key.raw_bytes(), - ) - .map_err(|e| data_error(e.to_string()))?; + )?; - let bytes_consumed = public_key - .encoded_len() - .map_err(|e| data_error(e.to_string()))?; + let bytes_consumed = public_key.encoded_len()?; if bytes_consumed != rsa::pkcs1::der::Length::new( pk_info.subject_public_key.raw_bytes().len() as u16, ) { - return Err(data_error("public key is invalid (too long)")); + return Err(ImportKeyError::PublicKeyTooLong); } let data = pk_info.subject_public_key.raw_bytes().to_vec().into(); @@ -435,30 +478,26 @@ fn import_key_rsaoaep( } KeyData::Pkcs8(data) => { // 2-3. - let pk_info = PrivateKeyInfo::from_der(&data) - .map_err(|e| data_error(e.to_string()))?; + let pk_info = PrivateKeyInfo::from_der(&data)?; // 4-5. let alg = pk_info.algorithm.oid; // 6-7. (skipped, only support rsaEncryption for interoperability) if alg != RSA_ENCRYPTION_OID { - return Err(data_error("unsupported algorithm")); + return Err(ImportKeyError::UnsupportedAlgorithm); } // 8-9. let private_key = - rsa::pkcs1::RsaPrivateKey::from_der(pk_info.private_key) - .map_err(|e| data_error(e.to_string()))?; + rsa::pkcs1::RsaPrivateKey::from_der(pk_info.private_key)?; - let bytes_consumed = private_key - .encoded_len() - .map_err(|e| data_error(e.to_string()))?; + let bytes_consumed = private_key.encoded_len()?; if bytes_consumed != rsa::pkcs1::der::Length::new(pk_info.private_key.len() as u16) { - return Err(data_error("private key is invalid (too long)")); + return Err(ImportKeyError::PrivateKeyTooLong); } let data = pk_info.private_key.to_vec().into(); @@ -475,14 +514,14 @@ fn import_key_rsaoaep( KeyData::JwkPublicRsa { .. } | KeyData::JwkPrivateRsa { .. } => { import_key_rsa_jwk(key_data) } - _ => Err(unsupported_format()), + _ => Err(SharedError::UnsupportedFormat.into()), } } fn decode_b64url_to_field_bytes( b64: &str, -) -> Result, deno_core::anyhow::Error> { - jwt_b64_int_or_err!(val, b64, "invalid b64 coordinate"); +) -> Result, ImportKeyError> { + jwt_b64_int_or_err!(val, b64, InvalidB64Coordinate); let mut bytes = elliptic_curve::FieldBytes::::default(); let original_bytes = val.as_bytes(); @@ -495,7 +534,7 @@ fn decode_b64url_to_field_bytes( let val = new_bytes.as_slice(); if val.len() != bytes.len() { - return Err(data_error("invalid b64 coordinate")); + return Err(ImportKeyError::InvalidB64Coordinate); } bytes.copy_from_slice(val); @@ -506,7 +545,7 @@ fn import_key_ec_jwk_to_point( x: String, y: String, named_curve: EcNamedCurve, -) -> Result, deno_core::anyhow::Error> { +) -> Result, ImportKeyError> { let point_bytes = match named_curve { EcNamedCurve::P256 => { let x = decode_b64url_to_field_bytes::(&x)?; @@ -534,7 +573,7 @@ fn import_key_ec_jwk_to_point( fn import_key_ec_jwk( key_data: KeyData, named_curve: EcNamedCurve, -) -> Result { +) -> Result { match key_data { KeyData::JwkPublicEc { x, y } => { let point_bytes = import_key_ec_jwk_to_point(x, y, named_curve)?; @@ -550,21 +589,21 @@ fn import_key_ec_jwk( let pk = p256::SecretKey::from_bytes(&d)?; pk.to_pkcs8_der() - .map_err(|_| data_error("invalid JWK private key"))? + .map_err(|_| ImportKeyError::InvalidJWKPrivateKey)? } EcNamedCurve::P384 => { let d = decode_b64url_to_field_bytes::(&d)?; let pk = p384::SecretKey::from_bytes(&d)?; pk.to_pkcs8_der() - .map_err(|_| data_error("invalid JWK private key"))? + .map_err(|_| ImportKeyError::InvalidJWKPrivateKey)? } EcNamedCurve::P521 => { let d = decode_b64url_to_field_bytes::(&d)?; let pk = p521::SecretKey::from_bytes(&d)?; pk.to_pkcs8_der() - .map_err(|_| data_error("invalid JWK private key"))? + .map_err(|_| ImportKeyError::InvalidJWKPrivateKey)? } }; @@ -595,7 +634,7 @@ impl<'a> TryFrom> for ECParametersSpki { fn import_key_ec( key_data: KeyData, named_curve: EcNamedCurve, -) -> Result { +) -> Result { match key_data { KeyData::Raw(data) => { // The point is parsed and validated, ultimately the original data is @@ -604,28 +643,28 @@ fn import_key_ec( EcNamedCurve::P256 => { // 1-2. let point = p256::EncodedPoint::from_bytes(&data) - .map_err(|_| data_error("invalid P-256 elliptic curve point"))?; + .map_err(|_| ImportKeyError::InvalidP256ECPoint)?; // 3. if point.is_identity() { - return Err(data_error("invalid P-256 elliptic curve point")); + return Err(ImportKeyError::InvalidP256ECPoint); } } EcNamedCurve::P384 => { // 1-2. let point = p384::EncodedPoint::from_bytes(&data) - .map_err(|_| data_error("invalid P-384 elliptic curve point"))?; + .map_err(|_| ImportKeyError::InvalidP384ECPoint)?; // 3. if point.is_identity() { - return Err(data_error("invalid P-384 elliptic curve point")); + return Err(ImportKeyError::InvalidP384ECPoint); } } EcNamedCurve::P521 => { // 1-2. let point = p521::EncodedPoint::from_bytes(&data) - .map_err(|_| data_error("invalid P-521 elliptic curve point"))?; + .map_err(|_| ImportKeyError::InvalidP521ECPoint)?; // 3. if point.is_identity() { - return Err(data_error("invalid P-521 elliptic curve point")); + return Err(ImportKeyError::InvalidP521ECPoint); } } }; @@ -635,11 +674,11 @@ fn import_key_ec( } KeyData::Pkcs8(data) => { let pk = PrivateKeyInfo::from_der(data.as_ref()) - .map_err(|_| data_error("expected valid PKCS#8 data"))?; + .map_err(|_| ImportKeyError::ExpectedValidPkcs8Data)?; let named_curve_alg = pk .algorithm .parameters - .ok_or_else(|| data_error("malformed parameters"))? + .ok_or(ImportKeyError::MalformedParameters)? .try_into() .unwrap(); @@ -654,7 +693,7 @@ fn import_key_ec( }; if pk_named_curve != Some(named_curve) { - return Err(data_error("curve mismatch")); + return Err(ImportKeyError::CurveMismatch); } Ok(ImportKeyResult::Ec { @@ -663,14 +702,13 @@ fn import_key_ec( } KeyData::Spki(data) => { // 2-3. - let pk_info = spki::SubjectPublicKeyInfoRef::try_from(&*data) - .map_err(|e| data_error(e.to_string()))?; + let pk_info = spki::SubjectPublicKeyInfoRef::try_from(&*data)?; // 4. let alg = pk_info.algorithm.oid; // id-ecPublicKey if alg != elliptic_curve::ALGORITHM_OID { - return Err(data_error("unsupported algorithm")); + return Err(ImportKeyError::UnsupportedAlgorithm); } // 5-7. @@ -678,9 +716,9 @@ fn import_key_ec( pk_info .algorithm .parameters - .ok_or_else(|| data_error("malformed parameters"))?, + .ok_or(ImportKeyError::MalformedParameters)?, ) - .map_err(|_| data_error("malformed parameters"))?; + .map_err(|_| ImportKeyError::MalformedParameters)?; // 8-9. let named_curve_alg = params.named_curve_alg; @@ -704,36 +742,30 @@ fn import_key_ec( let bytes_consumed = match named_curve { EcNamedCurve::P256 => { - let point = - p256::EncodedPoint::from_bytes(&*encoded_key).map_err(|_| { - data_error("invalid P-256 elliptic curve SPKI data") - })?; + let point = p256::EncodedPoint::from_bytes(&*encoded_key) + .map_err(|_| ImportKeyError::InvalidP256ECSPKIData)?; if point.is_identity() { - return Err(data_error("invalid P-256 elliptic curve point")); + return Err(ImportKeyError::InvalidP256ECPoint); } point.as_bytes().len() } EcNamedCurve::P384 => { - let point = - p384::EncodedPoint::from_bytes(&*encoded_key).map_err(|_| { - data_error("invalid P-384 elliptic curve SPKI data") - })?; + let point = p384::EncodedPoint::from_bytes(&*encoded_key) + .map_err(|_| ImportKeyError::InvalidP384ECSPKIData)?; if point.is_identity() { - return Err(data_error("invalid P-384 elliptic curve point")); + return Err(ImportKeyError::InvalidP384ECPoint); } point.as_bytes().len() } EcNamedCurve::P521 => { - let point = - p521::EncodedPoint::from_bytes(&*encoded_key).map_err(|_| { - data_error("invalid P-521 elliptic curve SPKI data") - })?; + let point = p521::EncodedPoint::from_bytes(&*encoded_key) + .map_err(|_| ImportKeyError::InvalidP521ECSPKIData)?; if point.is_identity() { - return Err(data_error("invalid P-521 elliptic curve point")); + return Err(ImportKeyError::InvalidP521ECPoint); } point.as_bytes().len() @@ -741,15 +773,15 @@ fn import_key_ec( }; if bytes_consumed != pk_info.subject_public_key.raw_bytes().len() { - return Err(data_error("public key is invalid (too long)")); + return Err(ImportKeyError::PublicKeyTooLong); } // 11. if named_curve != pk_named_curve { - return Err(data_error("curve mismatch")); + return Err(ImportKeyError::CurveMismatch); } } else { - return Err(data_error("Unsupported named curve")); + return Err(ImportKeyError::UnsupportedNamedCurve); } Ok(ImportKeyResult::Ec { @@ -759,34 +791,38 @@ fn import_key_ec( KeyData::JwkPublicEc { .. } | KeyData::JwkPrivateEc { .. } => { import_key_ec_jwk(key_data, named_curve) } - _ => Err(unsupported_format()), + _ => Err(SharedError::UnsupportedFormat.into()), } } -fn import_key_aes(key_data: KeyData) -> Result { +fn import_key_aes( + key_data: KeyData, +) -> Result { Ok(match key_data { KeyData::JwkSecret { k } => { let data = BASE64_URL_SAFE_FORGIVING .decode(k) - .map_err(|_| data_error("invalid key data"))?; + .map_err(|_| ImportKeyError::InvalidKeyData)?; ImportKeyResult::Hmac { raw_data: RustRawKeyData::Secret(data.into()), } } - _ => return Err(unsupported_format()), + _ => return Err(SharedError::UnsupportedFormat.into()), }) } -fn import_key_hmac(key_data: KeyData) -> Result { +fn import_key_hmac( + key_data: KeyData, +) -> Result { Ok(match key_data { KeyData::JwkSecret { k } => { let data = BASE64_URL_SAFE_FORGIVING .decode(k) - .map_err(|_| data_error("invalid key data"))?; + .map_err(|_| ImportKeyError::InvalidKeyData)?; ImportKeyResult::Hmac { raw_data: RustRawKeyData::Secret(data.into()), } } - _ => return Err(unsupported_format()), + _ => return Err(SharedError::UnsupportedFormat.into()), }) } diff --git a/ext/crypto/lib.rs b/ext/crypto/lib.rs index 7aa3462c76bbc3..69dcd1413a0641 100644 --- a/ext/crypto/lib.rs +++ b/ext/crypto/lib.rs @@ -6,10 +6,7 @@ use aes_kw::KekAes256; use base64::prelude::BASE64_URL_SAFE_NO_PAD; use base64::Engine; -use deno_core::error::custom_error; use deno_core::error::not_supported; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::ToJsBuffer; @@ -17,7 +14,6 @@ use deno_core::unsync::spawn_blocking; use deno_core::JsBuffer; use deno_core::OpState; use serde::Deserialize; -use shared::operation_error; use p256::elliptic_curve::sec1::FromEncodedPoint; use p256::pkcs8::DecodePrivateKey; @@ -64,17 +60,27 @@ mod import_key; mod key; mod shared; mod x25519; +mod x448; pub use crate::decrypt::op_crypto_decrypt; +pub use crate::decrypt::DecryptError; +pub use crate::ed25519::Ed25519Error; pub use crate::encrypt::op_crypto_encrypt; +pub use crate::encrypt::EncryptError; pub use crate::export_key::op_crypto_export_key; +pub use crate::export_key::ExportKeyError; pub use crate::generate_key::op_crypto_generate_key; +pub use crate::generate_key::GenerateKeyError; pub use crate::import_key::op_crypto_import_key; +pub use crate::import_key::ImportKeyError; use crate::key::Algorithm; use crate::key::CryptoHash; use crate::key::CryptoNamedCurve; use crate::key::HkdfOutput; +pub use crate::shared::SharedError; use crate::shared::V8RawKeyData; +pub use crate::x25519::X25519Error; +pub use crate::x448::X448Error; deno_core::extension!(deno_crypto, deps = [ deno_webidl, deno_web ], @@ -98,6 +104,14 @@ deno_core::extension!(deno_crypto, x25519::op_crypto_derive_bits_x25519, x25519::op_crypto_import_spki_x25519, x25519::op_crypto_import_pkcs8_x25519, + x25519::op_crypto_export_spki_x25519, + x25519::op_crypto_export_pkcs8_x25519, + x448::op_crypto_generate_x448_keypair, + x448::op_crypto_derive_bits_x448, + x448::op_crypto_import_spki_x448, + x448::op_crypto_import_pkcs8_x448, + x448::op_crypto_export_spki_x448, + x448::op_crypto_export_pkcs8_x448, ed25519::op_crypto_generate_ed25519_keypair, ed25519::op_crypto_import_spki_ed25519, ed25519::op_crypto_import_pkcs8_ed25519, @@ -106,8 +120,6 @@ deno_core::extension!(deno_crypto, ed25519::op_crypto_export_spki_ed25519, ed25519::op_crypto_export_pkcs8_ed25519, ed25519::op_crypto_jwk_x_ed25519, - x25519::op_crypto_export_spki_x25519, - x25519::op_crypto_export_pkcs8_x25519, ], esm = [ "00_crypto.js" ], options = { @@ -120,11 +132,63 @@ deno_core::extension!(deno_crypto, }, ); +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error(transparent)] + General(#[from] SharedError), + #[error(transparent)] + JoinError(#[from] tokio::task::JoinError), + #[error(transparent)] + Der(#[from] rsa::pkcs1::der::Error), + #[error("Missing argument hash")] + MissingArgumentHash, + #[error("Missing argument saltLength")] + MissingArgumentSaltLength, + #[error("unsupported algorithm")] + UnsupportedAlgorithm, + #[error(transparent)] + KeyRejected(#[from] ring::error::KeyRejected), + #[error(transparent)] + RSA(#[from] rsa::Error), + #[error(transparent)] + Pkcs1(#[from] rsa::pkcs1::Error), + #[error(transparent)] + Unspecified(#[from] ring::error::Unspecified), + #[error("Invalid key format")] + InvalidKeyFormat, + #[error(transparent)] + P256Ecdsa(#[from] p256::ecdsa::Error), + #[error("Unexpected error decoding private key")] + DecodePrivateKey, + #[error("Missing argument publicKey")] + MissingArgumentPublicKey, + #[error("Missing argument namedCurve")] + MissingArgumentNamedCurve, + #[error("Missing argument info")] + MissingArgumentInfo, + #[error("The length provided for HKDF is too large")] + HKDFLengthTooLarge, + #[error(transparent)] + Base64Decode(#[from] base64::DecodeError), + #[error("Data must be multiple of 8 bytes")] + DataInvalidSize, + #[error("Invalid key length")] + InvalidKeyLength, + #[error("encryption error")] + EncryptionError, + #[error("decryption error - integrity check failed")] + DecryptionError, + #[error("The ArrayBufferView's byte length ({0}) exceeds the number of bytes of entropy available via this API (65536)")] + ArrayBufferViewLengthExceeded(usize), + #[error(transparent)] + Other(deno_core::error::AnyError), +} + #[op2] #[serde] pub fn op_crypto_base64url_decode( #[string] data: String, -) -> Result { +) -> Result { let data: Vec = BASE64_URL_SAFE_NO_PAD.decode(data)?; Ok(data.into()) } @@ -140,12 +204,9 @@ pub fn op_crypto_base64url_encode(#[buffer] data: JsBuffer) -> String { pub fn op_crypto_get_random_values( state: &mut OpState, #[buffer] out: &mut [u8], -) -> Result<(), AnyError> { +) -> Result<(), Error> { if out.len() > 65536 { - return Err( - deno_web::DomExceptionQuotaExceededError::new(&format!("The ArrayBufferView's byte length ({}) exceeds the number of bytes of entropy available via this API (65536)", out.len())) - .into(), - ); + return Err(Error::ArrayBufferViewLengthExceeded(out.len())); } let maybe_seeded_rng = state.try_borrow_mut::(); @@ -197,7 +258,7 @@ pub struct SignArg { pub async fn op_crypto_sign_key( #[serde] args: SignArg, #[buffer] zero_copy: JsBuffer, -) -> Result { +) -> Result { deno_core::unsync::spawn_blocking(move || { let data = &*zero_copy; let algorithm = args.algorithm; @@ -206,10 +267,7 @@ pub async fn op_crypto_sign_key( Algorithm::RsassaPkcs1v15 => { use rsa::pkcs1v15::SigningKey; let private_key = RsaPrivateKey::from_pkcs1_der(&args.key.data)?; - match args - .hash - .ok_or_else(|| type_error("Missing argument hash".to_string()))? - { + match args.hash.ok_or_else(|| Error::MissingArgumentHash)? { CryptoHash::Sha1 => { let signing_key = SigningKey::::new(private_key); signing_key.sign(data) @@ -232,15 +290,13 @@ pub async fn op_crypto_sign_key( Algorithm::RsaPss => { let private_key = RsaPrivateKey::from_pkcs1_der(&args.key.data)?; - let salt_len = args.salt_length.ok_or_else(|| { - type_error("Missing argument saltLength".to_string()) - })? as usize; + let salt_len = args + .salt_length + .ok_or_else(|| Error::MissingArgumentSaltLength)? + as usize; let mut rng = OsRng; - match args - .hash - .ok_or_else(|| type_error("Missing argument hash".to_string()))? - { + match args.hash.ok_or_else(|| Error::MissingArgumentHash)? { CryptoHash::Sha1 => { let signing_key = Pss::new_with_salt::(salt_len); let hashed = Sha1::digest(data); @@ -265,8 +321,10 @@ pub async fn op_crypto_sign_key( .to_vec() } Algorithm::Ecdsa => { - let curve: &EcdsaSigningAlgorithm = - args.named_curve.ok_or_else(not_supported)?.into(); + let curve: &EcdsaSigningAlgorithm = args + .named_curve + .ok_or_else(|| Error::Other(not_supported()))? + .into(); let rng = RingRand::SystemRandom::new(); let key_pair = EcdsaKeyPair::from_pkcs8(curve, &args.key.data, &rng)?; @@ -275,7 +333,7 @@ pub async fn op_crypto_sign_key( if let Some(hash) = args.hash { match hash { CryptoHash::Sha256 | CryptoHash::Sha384 => (), - _ => return Err(type_error("Unsupported algorithm")), + _ => return Err(Error::UnsupportedAlgorithm), } }; @@ -285,14 +343,17 @@ pub async fn op_crypto_sign_key( signature.as_ref().to_vec() } Algorithm::Hmac => { - let hash: HmacAlgorithm = args.hash.ok_or_else(not_supported)?.into(); + let hash: HmacAlgorithm = args + .hash + .ok_or_else(|| Error::Other(not_supported()))? + .into(); let key = HmacKey::new(hash, &args.key.data); let signature = ring::hmac::sign(&key, data); signature.as_ref().to_vec() } - _ => return Err(type_error("Unsupported algorithm".to_string())), + _ => return Err(Error::UnsupportedAlgorithm), }; Ok(signature.into()) @@ -315,7 +376,7 @@ pub struct VerifyArg { pub async fn op_crypto_verify_key( #[serde] args: VerifyArg, #[buffer] zero_copy: JsBuffer, -) -> Result { +) -> Result { deno_core::unsync::spawn_blocking(move || { let data = &*zero_copy; let algorithm = args.algorithm; @@ -326,10 +387,7 @@ pub async fn op_crypto_verify_key( use rsa::pkcs1v15::VerifyingKey; let public_key = read_rsa_public_key(args.key)?; let signature: Signature = args.signature.as_ref().try_into()?; - match args - .hash - .ok_or_else(|| type_error("Missing argument hash".to_string()))? - { + match args.hash.ok_or_else(|| Error::MissingArgumentHash)? { CryptoHash::Sha1 => { let verifying_key = VerifyingKey::::new(public_key); verifying_key.verify(data, &signature).is_ok() @@ -352,14 +410,12 @@ pub async fn op_crypto_verify_key( let public_key = read_rsa_public_key(args.key)?; let signature = args.signature.as_ref(); - let salt_len = args.salt_length.ok_or_else(|| { - type_error("Missing argument saltLength".to_string()) - })? as usize; + let salt_len = args + .salt_length + .ok_or_else(|| Error::MissingArgumentSaltLength)? + as usize; - match args - .hash - .ok_or_else(|| type_error("Missing argument hash".to_string()))? - { + match args.hash.ok_or_else(|| Error::MissingArgumentHash)? { CryptoHash::Sha1 => { let pss = Pss::new_with_salt::(salt_len); let hashed = Sha1::digest(data); @@ -383,15 +439,22 @@ pub async fn op_crypto_verify_key( } } Algorithm::Hmac => { - let hash: HmacAlgorithm = args.hash.ok_or_else(not_supported)?.into(); + let hash: HmacAlgorithm = args + .hash + .ok_or_else(|| Error::Other(not_supported()))? + .into(); let key = HmacKey::new(hash, &args.key.data); ring::hmac::verify(&key, data, &args.signature).is_ok() } Algorithm::Ecdsa => { - let signing_alg: &EcdsaSigningAlgorithm = - args.named_curve.ok_or_else(not_supported)?.into(); - let verify_alg: &EcdsaVerificationAlgorithm = - args.named_curve.ok_or_else(not_supported)?.into(); + let signing_alg: &EcdsaSigningAlgorithm = args + .named_curve + .ok_or_else(|| Error::Other(not_supported()))? + .into(); + let verify_alg: &EcdsaVerificationAlgorithm = args + .named_curve + .ok_or_else(|| Error::Other(not_supported()))? + .into(); let private_key; @@ -404,7 +467,7 @@ pub async fn op_crypto_verify_key( private_key.public_key().as_ref() } KeyType::Public => &*args.key.data, - _ => return Err(type_error("Invalid Key format".to_string())), + _ => return Err(Error::InvalidKeyFormat), }; let public_key = @@ -412,7 +475,7 @@ pub async fn op_crypto_verify_key( public_key.verify(data, &args.signature).is_ok() } - _ => return Err(type_error("Unsupported algorithm".to_string())), + _ => return Err(Error::UnsupportedAlgorithm), }; Ok(verification) @@ -440,70 +503,68 @@ pub struct DeriveKeyArg { pub async fn op_crypto_derive_bits( #[serde] args: DeriveKeyArg, #[buffer] zero_copy: Option, -) -> Result { +) -> Result { deno_core::unsync::spawn_blocking(move || { let algorithm = args.algorithm; match algorithm { Algorithm::Pbkdf2 => { - let zero_copy = zero_copy.ok_or_else(not_supported)?; + let zero_copy = + zero_copy.ok_or_else(|| Error::Other(not_supported()))?; let salt = &*zero_copy; // The caller must validate these cases. assert!(args.length > 0); assert!(args.length % 8 == 0); - let algorithm = match args.hash.ok_or_else(not_supported)? { - CryptoHash::Sha1 => pbkdf2::PBKDF2_HMAC_SHA1, - CryptoHash::Sha256 => pbkdf2::PBKDF2_HMAC_SHA256, - CryptoHash::Sha384 => pbkdf2::PBKDF2_HMAC_SHA384, - CryptoHash::Sha512 => pbkdf2::PBKDF2_HMAC_SHA512, - }; + let algorithm = + match args.hash.ok_or_else(|| Error::Other(not_supported()))? { + CryptoHash::Sha1 => pbkdf2::PBKDF2_HMAC_SHA1, + CryptoHash::Sha256 => pbkdf2::PBKDF2_HMAC_SHA256, + CryptoHash::Sha384 => pbkdf2::PBKDF2_HMAC_SHA384, + CryptoHash::Sha512 => pbkdf2::PBKDF2_HMAC_SHA512, + }; // This will never panic. We have already checked length earlier. - let iterations = - NonZeroU32::new(args.iterations.ok_or_else(not_supported)?).unwrap(); + let iterations = NonZeroU32::new( + args + .iterations + .ok_or_else(|| Error::Other(not_supported()))?, + ) + .unwrap(); let secret = args.key.data; let mut out = vec![0; args.length / 8]; pbkdf2::derive(algorithm, iterations, salt, &secret, &mut out); Ok(out.into()) } Algorithm::Ecdh => { - let named_curve = args.named_curve.ok_or_else(|| { - type_error("Missing argument namedCurve".to_string()) - })?; + let named_curve = args + .named_curve + .ok_or_else(|| Error::MissingArgumentNamedCurve)?; let public_key = args .public_key - .ok_or_else(|| type_error("Missing argument publicKey"))?; + .ok_or_else(|| Error::MissingArgumentPublicKey)?; match named_curve { CryptoNamedCurve::P256 => { let secret_key = p256::SecretKey::from_pkcs8_der(&args.key.data) - .map_err(|_| { - type_error("Unexpected error decoding private key") - })?; + .map_err(|_| Error::DecodePrivateKey)?; let public_key = match public_key.r#type { KeyType::Private => { p256::SecretKey::from_pkcs8_der(&public_key.data) - .map_err(|_| { - type_error("Unexpected error decoding private key") - })? + .map_err(|_| Error::DecodePrivateKey)? .public_key() } KeyType::Public => { let point = p256::EncodedPoint::from_bytes(public_key.data) - .map_err(|_| { - type_error("Unexpected error decoding private key") - })?; + .map_err(|_| Error::DecodePrivateKey)?; let pk = p256::PublicKey::from_encoded_point(&point); // pk is a constant time Option. if pk.is_some().into() { pk.unwrap() } else { - return Err(type_error( - "Unexpected error decoding private key", - )); + return Err(Error::DecodePrivateKey); } } _ => unreachable!(), @@ -519,32 +580,24 @@ pub async fn op_crypto_derive_bits( } CryptoNamedCurve::P384 => { let secret_key = p384::SecretKey::from_pkcs8_der(&args.key.data) - .map_err(|_| { - type_error("Unexpected error decoding private key") - })?; + .map_err(|_| Error::DecodePrivateKey)?; let public_key = match public_key.r#type { KeyType::Private => { p384::SecretKey::from_pkcs8_der(&public_key.data) - .map_err(|_| { - type_error("Unexpected error decoding private key") - })? + .map_err(|_| Error::DecodePrivateKey)? .public_key() } KeyType::Public => { let point = p384::EncodedPoint::from_bytes(public_key.data) - .map_err(|_| { - type_error("Unexpected error decoding private key") - })?; + .map_err(|_| Error::DecodePrivateKey)?; let pk = p384::PublicKey::from_encoded_point(&point); // pk is a constant time Option. if pk.is_some().into() { pk.unwrap() } else { - return Err(type_error( - "Unexpected error decoding private key", - )); + return Err(Error::DecodePrivateKey); } } _ => unreachable!(), @@ -561,18 +614,18 @@ pub async fn op_crypto_derive_bits( } } Algorithm::Hkdf => { - let zero_copy = zero_copy.ok_or_else(not_supported)?; + let zero_copy = + zero_copy.ok_or_else(|| Error::Other(not_supported()))?; let salt = &*zero_copy; - let algorithm = match args.hash.ok_or_else(not_supported)? { - CryptoHash::Sha1 => hkdf::HKDF_SHA1_FOR_LEGACY_USE_ONLY, - CryptoHash::Sha256 => hkdf::HKDF_SHA256, - CryptoHash::Sha384 => hkdf::HKDF_SHA384, - CryptoHash::Sha512 => hkdf::HKDF_SHA512, - }; - - let info = args - .info - .ok_or_else(|| type_error("Missing argument info".to_string()))?; + let algorithm = + match args.hash.ok_or_else(|| Error::Other(not_supported()))? { + CryptoHash::Sha1 => hkdf::HKDF_SHA1_FOR_LEGACY_USE_ONLY, + CryptoHash::Sha256 => hkdf::HKDF_SHA256, + CryptoHash::Sha384 => hkdf::HKDF_SHA384, + CryptoHash::Sha512 => hkdf::HKDF_SHA512, + }; + + let info = args.info.ok_or_else(|| Error::MissingArgumentInfo)?; // IKM let secret = args.key.data; // L @@ -581,23 +634,20 @@ pub async fn op_crypto_derive_bits( let salt = hkdf::Salt::new(algorithm, salt); let prk = salt.extract(&secret); let info = &[&*info]; - let okm = prk.expand(info, HkdfOutput(length)).map_err(|_e| { - custom_error( - "DOMExceptionOperationError", - "The length provided for HKDF is too large", - ) - })?; + let okm = prk + .expand(info, HkdfOutput(length)) + .map_err(|_e| Error::HKDFLengthTooLarge)?; let mut r = vec![0u8; length]; okm.fill(&mut r)?; Ok(r.into()) } - _ => Err(type_error("Unsupported algorithm".to_string())), + _ => Err(Error::UnsupportedAlgorithm), } }) .await? } -fn read_rsa_public_key(key_data: KeyData) -> Result { +fn read_rsa_public_key(key_data: KeyData) -> Result { let public_key = match key_data.r#type { KeyType::Private => { RsaPrivateKey::from_pkcs1_der(&key_data.data)?.to_public_key() @@ -610,7 +660,7 @@ fn read_rsa_public_key(key_data: KeyData) -> Result { #[op2] #[string] -pub fn op_crypto_random_uuid(state: &mut OpState) -> Result { +pub fn op_crypto_random_uuid(state: &mut OpState) -> Result { let maybe_seeded_rng = state.try_borrow_mut::(); let uuid = if let Some(seeded_rng) = maybe_seeded_rng { let mut bytes = [0u8; 16]; @@ -631,7 +681,7 @@ pub fn op_crypto_random_uuid(state: &mut OpState) -> Result { pub async fn op_crypto_subtle_digest( #[serde] algorithm: CryptoHash, #[buffer] data: JsBuffer, -) -> Result { +) -> Result { let output = spawn_blocking(move || { digest::digest(algorithm.into(), &data) .as_ref() @@ -655,7 +705,7 @@ pub struct WrapUnwrapKeyArg { pub fn op_crypto_wrap_key( #[serde] args: WrapUnwrapKeyArg, #[buffer] data: JsBuffer, -) -> Result { +) -> Result { let algorithm = args.algorithm; match algorithm { @@ -663,20 +713,20 @@ pub fn op_crypto_wrap_key( let key = args.key.as_secret_key()?; if data.len() % 8 != 0 { - return Err(type_error("Data must be multiple of 8 bytes")); + return Err(Error::DataInvalidSize); } let wrapped_key = match key.len() { 16 => KekAes128::new(key.into()).wrap_vec(&data), 24 => KekAes192::new(key.into()).wrap_vec(&data), 32 => KekAes256::new(key.into()).wrap_vec(&data), - _ => return Err(type_error("Invalid key length")), + _ => return Err(Error::InvalidKeyLength), } - .map_err(|_| operation_error("encryption error"))?; + .map_err(|_| Error::EncryptionError)?; Ok(wrapped_key.into()) } - _ => Err(type_error("Unsupported algorithm")), + _ => Err(Error::UnsupportedAlgorithm), } } @@ -685,29 +735,27 @@ pub fn op_crypto_wrap_key( pub fn op_crypto_unwrap_key( #[serde] args: WrapUnwrapKeyArg, #[buffer] data: JsBuffer, -) -> Result { +) -> Result { let algorithm = args.algorithm; match algorithm { Algorithm::AesKw => { let key = args.key.as_secret_key()?; if data.len() % 8 != 0 { - return Err(type_error("Data must be multiple of 8 bytes")); + return Err(Error::DataInvalidSize); } let unwrapped_key = match key.len() { 16 => KekAes128::new(key.into()).unwrap_vec(&data), 24 => KekAes192::new(key.into()).unwrap_vec(&data), 32 => KekAes256::new(key.into()).unwrap_vec(&data), - _ => return Err(type_error("Invalid key length")), + _ => return Err(Error::InvalidKeyLength), } - .map_err(|_| { - operation_error("decryption error - integrity check failed") - })?; + .map_err(|_| Error::DecryptionError)?; Ok(unwrapped_key.into()) } - _ => Err(type_error("Unsupported algorithm")), + _ => Err(Error::UnsupportedAlgorithm), } } diff --git a/ext/crypto/shared.rs b/ext/crypto/shared.rs index d06a268cd6ca72..f70d32856ccb40 100644 --- a/ext/crypto/shared.rs +++ b/ext/crypto/shared.rs @@ -2,9 +2,6 @@ use std::borrow::Cow; -use deno_core::error::custom_error; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::JsBuffer; use deno_core::ToJsBuffer; use elliptic_curve::sec1::ToEncodedPoint; @@ -63,47 +60,73 @@ pub enum RustRawKeyData { Public(ToJsBuffer), } +#[derive(Debug, thiserror::Error)] +pub enum SharedError { + #[error("expected valid private key")] + ExpectedValidPrivateKey, + #[error("expected valid public key")] + ExpectedValidPublicKey, + #[error("expected valid private EC key")] + ExpectedValidPrivateECKey, + #[error("expected valid public EC key")] + ExpectedValidPublicECKey, + #[error("expected private key")] + ExpectedPrivateKey, + #[error("expected public key")] + ExpectedPublicKey, + #[error("expected secret key")] + ExpectedSecretKey, + #[error("failed to decode private key")] + FailedDecodePrivateKey, + #[error("failed to decode public key")] + FailedDecodePublicKey, + #[error("unsupported format")] + UnsupportedFormat, +} + impl V8RawKeyData { - pub fn as_rsa_public_key(&self) -> Result, AnyError> { + pub fn as_rsa_public_key(&self) -> Result, SharedError> { match self { V8RawKeyData::Public(data) => Ok(Cow::Borrowed(data)), V8RawKeyData::Private(data) => { let private_key = RsaPrivateKey::from_pkcs1_der(data) - .map_err(|_| type_error("expected valid private key"))?; + .map_err(|_| SharedError::ExpectedValidPrivateKey)?; let public_key_doc = private_key .to_public_key() .to_pkcs1_der() - .map_err(|_| type_error("expected valid public key"))?; + .map_err(|_| SharedError::ExpectedValidPublicKey)?; Ok(Cow::Owned(public_key_doc.as_bytes().into())) } - _ => Err(type_error("expected public key")), + _ => Err(SharedError::ExpectedPublicKey), } } - pub fn as_rsa_private_key(&self) -> Result<&[u8], AnyError> { + pub fn as_rsa_private_key(&self) -> Result<&[u8], SharedError> { match self { V8RawKeyData::Private(data) => Ok(data), - _ => Err(type_error("expected private key")), + _ => Err(SharedError::ExpectedPrivateKey), } } - pub fn as_secret_key(&self) -> Result<&[u8], AnyError> { + pub fn as_secret_key(&self) -> Result<&[u8], SharedError> { match self { V8RawKeyData::Secret(data) => Ok(data), - _ => Err(type_error("expected secret key")), + _ => Err(SharedError::ExpectedSecretKey), } } - pub fn as_ec_public_key_p256(&self) -> Result { + pub fn as_ec_public_key_p256( + &self, + ) -> Result { match self { V8RawKeyData::Public(data) => p256::PublicKey::from_sec1_bytes(data) .map(|p| p.to_encoded_point(false)) - .map_err(|_| type_error("expected valid public EC key")), + .map_err(|_| SharedError::ExpectedValidPublicECKey), V8RawKeyData::Private(data) => { let signing_key = p256::SecretKey::from_pkcs8_der(data) - .map_err(|_| type_error("expected valid private EC key"))?; + .map_err(|_| SharedError::ExpectedValidPrivateECKey)?; Ok(signing_key.public_key().to_encoded_point(false)) } // Should never reach here. @@ -111,14 +134,16 @@ impl V8RawKeyData { } } - pub fn as_ec_public_key_p384(&self) -> Result { + pub fn as_ec_public_key_p384( + &self, + ) -> Result { match self { V8RawKeyData::Public(data) => p384::PublicKey::from_sec1_bytes(data) .map(|p| p.to_encoded_point(false)) - .map_err(|_| type_error("expected valid public EC key")), + .map_err(|_| SharedError::ExpectedValidPublicECKey), V8RawKeyData::Private(data) => { let signing_key = p384::SecretKey::from_pkcs8_der(data) - .map_err(|_| type_error("expected valid private EC key"))?; + .map_err(|_| SharedError::ExpectedValidPrivateECKey)?; Ok(signing_key.public_key().to_encoded_point(false)) } // Should never reach here. @@ -126,16 +151,18 @@ impl V8RawKeyData { } } - pub fn as_ec_public_key_p521(&self) -> Result { + pub fn as_ec_public_key_p521( + &self, + ) -> Result { match self { V8RawKeyData::Public(data) => { // public_key is a serialized EncodedPoint p521::EncodedPoint::from_bytes(data) - .map_err(|_| type_error("expected valid public EC key")) + .map_err(|_| SharedError::ExpectedValidPublicECKey) } V8RawKeyData::Private(data) => { let signing_key = p521::SecretKey::from_pkcs8_der(data) - .map_err(|_| type_error("expected valid private EC key"))?; + .map_err(|_| SharedError::ExpectedValidPrivateECKey)?; Ok(signing_key.public_key().to_encoded_point(false)) } // Should never reach here. @@ -143,26 +170,10 @@ impl V8RawKeyData { } } - pub fn as_ec_private_key(&self) -> Result<&[u8], AnyError> { + pub fn as_ec_private_key(&self) -> Result<&[u8], SharedError> { match self { V8RawKeyData::Private(data) => Ok(data), - _ => Err(type_error("expected private key")), + _ => Err(SharedError::ExpectedPrivateKey), } } } - -pub fn data_error(msg: impl Into>) -> AnyError { - custom_error("DOMExceptionDataError", msg) -} - -pub fn not_supported_error(msg: impl Into>) -> AnyError { - custom_error("DOMExceptionNotSupportedError", msg) -} - -pub fn operation_error(msg: impl Into>) -> AnyError { - custom_error("DOMExceptionOperationError", msg) -} - -pub fn unsupported_format() -> AnyError { - not_supported_error("unsupported format") -} diff --git a/ext/crypto/x25519.rs b/ext/crypto/x25519.rs index 8fcad3ef213289..d2c4d986b98dec 100644 --- a/ext/crypto/x25519.rs +++ b/ext/crypto/x25519.rs @@ -1,8 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use curve25519_dalek::montgomery::MontgomeryPoint; -use deno_core::error::custom_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::ToJsBuffer; use elliptic_curve::pkcs8::PrivateKeyInfo; @@ -13,6 +11,14 @@ use spki::der::asn1::BitString; use spki::der::Decode; use spki::der::Encode; +#[derive(Debug, thiserror::Error)] +pub enum X25519Error { + #[error("Failed to export key")] + FailedExport, + #[error(transparent)] + Der(#[from] spki::der::Error), +} + #[op2(fast)] pub fn op_crypto_generate_x25519_keypair( #[buffer] pkey: &mut [u8], @@ -47,10 +53,10 @@ pub fn op_crypto_derive_bits_x25519( let sh_sec = x25519_dalek::x25519(k, u); let point = MontgomeryPoint(sh_sec); if point.ct_eq(&MONTGOMERY_IDENTITY).unwrap_u8() == 1 { - return false; + return true; } secret.copy_from_slice(&sh_sec); - true + false } // id-X25519 OBJECT IDENTIFIER ::= { 1 3 101 110 } @@ -113,7 +119,7 @@ pub fn op_crypto_import_pkcs8_x25519( #[serde] pub fn op_crypto_export_spki_x25519( #[buffer] pubkey: &[u8], -) -> Result { +) -> Result { let key_info = spki::SubjectPublicKeyInfo { algorithm: spki::AlgorithmIdentifierRef { // id-X25519 @@ -125,9 +131,7 @@ pub fn op_crypto_export_spki_x25519( Ok( key_info .to_der() - .map_err(|_| { - custom_error("DOMExceptionOperationError", "Failed to export key") - })? + .map_err(|_| X25519Error::FailedExport)? .into(), ) } @@ -136,7 +140,7 @@ pub fn op_crypto_export_spki_x25519( #[serde] pub fn op_crypto_export_pkcs8_x25519( #[buffer] pkey: &[u8], -) -> Result { +) -> Result { use rsa::pkcs1::der::Encode; // This should probably use OneAsymmetricKey instead diff --git a/ext/crypto/x448.rs b/ext/crypto/x448.rs new file mode 100644 index 00000000000000..89bf48e28bd1c0 --- /dev/null +++ b/ext/crypto/x448.rs @@ -0,0 +1,152 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use deno_core::op2; +use deno_core::ToJsBuffer; +use ed448_goldilocks::curve::MontgomeryPoint; +use ed448_goldilocks::Scalar; +use elliptic_curve::pkcs8::PrivateKeyInfo; +use elliptic_curve::subtle::ConstantTimeEq; +use rand::rngs::OsRng; +use rand::RngCore; +use spki::der::asn1::BitString; +use spki::der::Decode; +use spki::der::Encode; + +#[derive(Debug, thiserror::Error)] +pub enum X448Error { + #[error("Failed to export key")] + FailedExport, + #[error(transparent)] + Der(#[from] spki::der::Error), +} + +#[op2(fast)] +pub fn op_crypto_generate_x448_keypair( + #[buffer] pkey: &mut [u8], + #[buffer] pubkey: &mut [u8], +) { + let mut rng = OsRng; + rng.fill_bytes(pkey); + + // x448(pkey, 5) + let point = &MontgomeryPoint::generator() + * &Scalar::from_bytes(pkey.try_into().unwrap()); + pubkey.copy_from_slice(&point.0); +} + +const MONTGOMERY_IDENTITY: MontgomeryPoint = MontgomeryPoint([0; 56]); + +#[op2(fast)] +pub fn op_crypto_derive_bits_x448( + #[buffer] k: &[u8], + #[buffer] u: &[u8], + #[buffer] secret: &mut [u8], +) -> bool { + let k: [u8; 56] = k.try_into().expect("Expected byteLength 56"); + let u: [u8; 56] = u.try_into().expect("Expected byteLength 56"); + + // x448(k, u) + let point = &MontgomeryPoint(u) * &Scalar::from_bytes(k); + if point.ct_eq(&MONTGOMERY_IDENTITY).unwrap_u8() == 1 { + return true; + } + + secret.copy_from_slice(&point.0); + false +} + +// id-X448 OBJECT IDENTIFIER ::= { 1 3 101 111 } +const X448_OID: const_oid::ObjectIdentifier = + const_oid::ObjectIdentifier::new_unwrap("1.3.101.111"); + +#[op2] +#[serde] +pub fn op_crypto_export_spki_x448( + #[buffer] pubkey: &[u8], +) -> Result { + let key_info = spki::SubjectPublicKeyInfo { + algorithm: spki::AlgorithmIdentifierRef { + oid: X448_OID, + parameters: None, + }, + subject_public_key: BitString::from_bytes(pubkey)?, + }; + Ok( + key_info + .to_der() + .map_err(|_| X448Error::FailedExport)? + .into(), + ) +} + +#[op2] +#[serde] +pub fn op_crypto_export_pkcs8_x448( + #[buffer] pkey: &[u8], +) -> Result { + use rsa::pkcs1::der::Encode; + + let pk_info = rsa::pkcs8::PrivateKeyInfo { + public_key: None, + algorithm: rsa::pkcs8::AlgorithmIdentifierRef { + oid: X448_OID, + parameters: None, + }, + private_key: pkey, // OCTET STRING + }; + + let mut buf = Vec::new(); + pk_info.encode_to_vec(&mut buf)?; + Ok(buf.into()) +} + +#[op2(fast)] +pub fn op_crypto_import_spki_x448( + #[buffer] key_data: &[u8], + #[buffer] out: &mut [u8], +) -> bool { + // 2-3. + let pk_info = match spki::SubjectPublicKeyInfoRef::try_from(key_data) { + Ok(pk_info) => pk_info, + Err(_) => return false, + }; + // 4. + let alg = pk_info.algorithm.oid; + if alg != X448_OID { + return false; + } + // 5. + if pk_info.algorithm.parameters.is_some() { + return false; + } + out.copy_from_slice(pk_info.subject_public_key.raw_bytes()); + true +} + +#[op2(fast)] +pub fn op_crypto_import_pkcs8_x448( + #[buffer] key_data: &[u8], + #[buffer] out: &mut [u8], +) -> bool { + // 2-3. + let pk_info = match PrivateKeyInfo::from_der(key_data) { + Ok(pk_info) => pk_info, + Err(_) => return false, + }; + // 4. + let alg = pk_info.algorithm.oid; + if alg != X448_OID { + return false; + } + // 5. + if pk_info.algorithm.parameters.is_some() { + return false; + } + // 6. + // CurvePrivateKey ::= OCTET STRING + if pk_info.private_key.len() != 56 { + return false; + } + out.copy_from_slice(&pk_info.private_key[2..]); + true +} diff --git a/ext/fetch/22_body.js b/ext/fetch/22_body.js index 61a06b4afddffd..a34758d19a532c 100644 --- a/ext/fetch/22_body.js +++ b/ext/fetch/22_body.js @@ -15,6 +15,7 @@ import { core, primordials } from "ext:core/mod.js"; const { isAnyArrayBuffer, isArrayBuffer, + isStringObject, } = core; const { ArrayBufferIsView, @@ -285,8 +286,13 @@ function mixinBody(prototype, bodySymbol, mimeTypeSymbol) { */ get() { webidl.assertBranded(this, prototype); - if (this[bodySymbol] !== null) { - return this[bodySymbol].consumed(); + try { + if (this[bodySymbol] !== null) { + return this[bodySymbol].consumed(); + } + } catch (_) { + // Request is closed. + return true; } return false; }, @@ -466,6 +472,8 @@ function extractBody(object) { if (object.locked || isReadableStreamDisturbed(object)) { throw new TypeError("ReadableStream is locked or disturbed"); } + } else if (object[webidl.AsyncIterable] === webidl.AsyncIterable) { + stream = ReadableStream.from(object.open()); } if (typeof source === "string") { // WARNING: this deviates from spec (expects length to be set) @@ -483,6 +491,9 @@ function extractBody(object) { return { body, contentType }; } +webidl.converters["async iterable"] = webidl + .createAsyncIterableConverter(webidl.converters.Uint8Array); + webidl.converters["BodyInit_DOMString"] = (V, prefix, context, opts) => { // Union for (ReadableStream or Blob or ArrayBufferView or ArrayBuffer or FormData or URLSearchParams or USVString) if (ObjectPrototypeIsPrototypeOf(ReadableStreamPrototype, V)) { @@ -501,6 +512,14 @@ webidl.converters["BodyInit_DOMString"] = (V, prefix, context, opts) => { if (ArrayBufferIsView(V)) { return webidl.converters["ArrayBufferView"](V, prefix, context, opts); } + if (webidl.isAsyncIterable(V) && !isStringObject(V)) { + return webidl.converters["async iterable"]( + V, + prefix, + context, + opts, + ); + } } // BodyInit conversion is passed to extractBody(), which calls core.encode(). // core.encode() will UTF-8 encode strings with replacement, being equivalent to the USV normalization. diff --git a/ext/fetch/23_request.js b/ext/fetch/23_request.js index 6211e927d92d1d..61cac22d2e2aaa 100644 --- a/ext/fetch/23_request.js +++ b/ext/fetch/23_request.js @@ -269,19 +269,25 @@ class Request { /** @type {AbortSignal} */ get [_signal]() { const signal = this[_signalCache]; - // This signal not been created yet, and the request is still in progress - if (signal === undefined) { + // This signal has not been created yet, but the request has already completed + if (signal === false) { const signal = newSignal(); this[_signalCache] = signal; + signal[signalAbort](signalAbortError); return signal; } - // This signal has not been created yet, but the request has already completed - if (signal === false) { + + // This signal not been created yet, and the request is still in progress + if (signal === undefined) { const signal = newSignal(); this[_signalCache] = signal; - signal[signalAbort](signalAbortError); + this[_request].onCancel?.(() => { + signal[signalAbort](signalAbortError); + }); + return signal; } + return signal; } get [_mimeType]() { diff --git a/ext/fetch/23_response.js b/ext/fetch/23_response.js index ff4ad5facd2d3c..278dcb7dec6300 100644 --- a/ext/fetch/23_response.js +++ b/ext/fetch/23_response.js @@ -61,6 +61,15 @@ const _mimeType = Symbol("mime type"); const _body = Symbol("body"); const _brand = webidl.brand; +// it's slightly faster to cache these +const webidlConvertersBodyInitDomString = + webidl.converters["BodyInit_DOMString?"]; +const webidlConvertersUSVString = webidl.converters["USVString"]; +const webidlConvertersUnsignedShort = webidl.converters["unsigned short"]; +const webidlConvertersAny = webidl.converters["any"]; +const webidlConvertersByteString = webidl.converters["ByteString"]; +const webidlConvertersHeadersInit = webidl.converters["HeadersInit"]; + /** * @typedef InnerResponse * @property {"basic" | "cors" | "default" | "error" | "opaque" | "opaqueredirect"} type @@ -259,8 +268,8 @@ class Response { */ static redirect(url, status = 302) { const prefix = "Failed to execute 'Response.redirect'"; - url = webidl.converters["USVString"](url, prefix, "Argument 1"); - status = webidl.converters["unsigned short"](status, prefix, "Argument 2"); + url = webidlConvertersUSVString(url, prefix, "Argument 1"); + status = webidlConvertersUnsignedShort(status, prefix, "Argument 2"); const baseURL = getLocationHref(); const parsedURL = new URL(url, baseURL); @@ -286,8 +295,8 @@ class Response { */ static json(data = undefined, init = { __proto__: null }) { const prefix = "Failed to execute 'Response.json'"; - data = webidl.converters.any(data); - init = webidl.converters["ResponseInit_fast"](init, prefix, "Argument 2"); + data = webidlConvertersAny(data); + init = webidlConvertersResponseInitFast(init, prefix, "Argument 2"); const str = serializeJSValueToJSONString(data); const res = extractBody(str); @@ -313,8 +322,8 @@ class Response { } const prefix = "Failed to construct 'Response'"; - body = webidl.converters["BodyInit_DOMString?"](body, prefix, "Argument 1"); - init = webidl.converters["ResponseInit_fast"](init, prefix, "Argument 2"); + body = webidlConvertersBodyInitDomString(body, prefix, "Argument 1"); + init = webidlConvertersResponseInitFast(init, prefix, "Argument 2"); this[_response] = newInnerResponse(); this[_headers] = headersFromHeaderList( @@ -443,47 +452,49 @@ webidl.converters["Response"] = webidl.createInterfaceConverter( "Response", ResponsePrototype, ); -webidl.converters["ResponseInit"] = webidl.createDictionaryConverter( - "ResponseInit", - [{ - key: "status", - defaultValue: 200, - converter: webidl.converters["unsigned short"], - }, { - key: "statusText", - defaultValue: "", - converter: webidl.converters["ByteString"], - }, { - key: "headers", - converter: webidl.converters["HeadersInit"], - }], -); -webidl.converters["ResponseInit_fast"] = function ( - init, - prefix, - context, - opts, -) { - if (init === undefined || init === null) { - return { status: 200, statusText: "", headers: undefined }; - } - // Fast path, if not a proxy - if (typeof init === "object" && !core.isProxy(init)) { - // Not a proxy fast path - const status = init.status !== undefined - ? webidl.converters["unsigned short"](init.status) - : 200; - const statusText = init.statusText !== undefined - ? webidl.converters["ByteString"](init.statusText) - : ""; - const headers = init.headers !== undefined - ? webidl.converters["HeadersInit"](init.headers) - : undefined; - return { status, statusText, headers }; - } - // Slow default path - return webidl.converters["ResponseInit"](init, prefix, context, opts); -}; +const webidlConvertersResponseInit = webidl.converters["ResponseInit"] = webidl + .createDictionaryConverter( + "ResponseInit", + [{ + key: "status", + defaultValue: 200, + converter: webidlConvertersUnsignedShort, + }, { + key: "statusText", + defaultValue: "", + converter: webidlConvertersByteString, + }, { + key: "headers", + converter: webidlConvertersHeadersInit, + }], + ); +const webidlConvertersResponseInitFast = webidl + .converters["ResponseInit_fast"] = function ( + init, + prefix, + context, + opts, + ) { + if (init === undefined || init === null) { + return { status: 200, statusText: "", headers: undefined }; + } + // Fast path, if not a proxy + if (typeof init === "object" && !core.isProxy(init)) { + // Not a proxy fast path + const status = init.status !== undefined + ? webidlConvertersUnsignedShort(init.status) + : 200; + const statusText = init.statusText !== undefined + ? webidlConvertersByteString(init.statusText) + : ""; + const headers = init.headers !== undefined + ? webidlConvertersHeadersInit(init.headers) + : undefined; + return { status, statusText, headers }; + } + // Slow default path + return webidlConvertersResponseInit(init, prefix, context, opts); + }; /** * @param {Response} response diff --git a/ext/fetch/26_fetch.js b/ext/fetch/26_fetch.js index 8ac364a931c90f..12b9c4582b7a1f 100644 --- a/ext/fetch/26_fetch.js +++ b/ext/fetch/26_fetch.js @@ -13,6 +13,7 @@ import { core, primordials } from "ext:core/mod.js"; import { op_fetch, + op_fetch_promise_is_settled, op_fetch_send, op_wasm_streaming_feed, op_wasm_streaming_set_url, @@ -28,7 +29,9 @@ const { PromisePrototypeThen, PromisePrototypeCatch, SafeArrayIterator, + SafePromisePrototypeFinally, String, + StringPrototypeEndsWith, StringPrototypeStartsWith, StringPrototypeToLowerCase, TypeError, @@ -55,6 +58,17 @@ import { toInnerResponse, } from "ext:deno_fetch/23_response.js"; import * as abortSignal from "ext:deno_web/03_abort_signal.js"; +import { + endSpan, + enterSpan, + exitSpan, + Span, + TRACING_ENABLED, +} from "ext:deno_telemetry/telemetry.ts"; +import { + updateSpanFromRequest, + updateSpanFromResponse, +} from "ext:deno_telemetry/util.ts"; const REQUEST_BODY_HEADER_NAMES = [ "content-encoding", @@ -63,6 +77,12 @@ const REQUEST_BODY_HEADER_NAMES = [ "content-type", ]; +const REDIRECT_SENSITIVE_HEADER_NAMES = [ + "authorization", + "proxy-authorization", + "cookie", +]; + /** * @param {number} rid * @returns {Promise<{ status: number, statusText: string, headers: [string, string][], url: string, responseRid: number, error: [string, string]? }>} @@ -250,12 +270,14 @@ function httpRedirectFetch(request, response, terminator) { if (locationHeaders.length === 0) { return response; } + + const currentURL = new URL(request.currentUrl()); const locationURL = new URL( locationHeaders[0][1], response.url() ?? undefined, ); if (locationURL.hash === "") { - locationURL.hash = request.currentUrl().hash; + locationURL.hash = currentURL.hash; } if (locationURL.protocol !== "https:" && locationURL.protocol !== "http:") { return networkError("Can not redirect to a non HTTP(s) url"); @@ -294,6 +316,28 @@ function httpRedirectFetch(request, response, terminator) { } } } + + // Drop confidential headers when redirecting to a less secure protocol + // or to a different domain that is not a superdomain + if ( + locationURL.protocol !== currentURL.protocol && + locationURL.protocol !== "https:" || + locationURL.host !== currentURL.host && + !isSubdomain(locationURL.host, currentURL.host) + ) { + for (let i = 0; i < request.headerList.length; i++) { + if ( + ArrayPrototypeIncludes( + REDIRECT_SENSITIVE_HEADER_NAMES, + byteLowerCase(request.headerList[i][0]), + ) + ) { + ArrayPrototypeSplice(request.headerList, i, 1); + i--; + } + } + } + if (request.body !== null) { const res = extractBody(request.body.source); request.body = res.body; @@ -307,93 +351,138 @@ function httpRedirectFetch(request, response, terminator) { * @param {RequestInit} init */ function fetch(input, init = { __proto__: null }) { - // There is an async dispatch later that causes a stack trace disconnect. - // We reconnect it by assigning the result of that dispatch to `opPromise`, - // awaiting `opPromise` in an inner function also named `fetch()` and - // returning the result from that. - let opPromise = undefined; - // 1. - const result = new Promise((resolve, reject) => { - const prefix = "Failed to execute 'fetch'"; - webidl.requiredArguments(arguments.length, 1, prefix); - // 2. - const requestObject = new Request(input, init); - // 3. - const request = toInnerRequest(requestObject); - // 4. - if (requestObject.signal.aborted) { - reject(abortFetch(request, null, requestObject.signal.reason)); - return; + let span; + try { + if (TRACING_ENABLED) { + span = new Span("fetch", { kind: 2 }); + enterSpan(span); } - // 7. - let responseObject = null; - // 9. - let locallyAborted = false; - // 10. - function onabort() { - locallyAborted = true; - reject( - abortFetch(request, responseObject, requestObject.signal.reason), - ); - } - requestObject.signal[abortSignal.add](onabort); + // There is an async dispatch later that causes a stack trace disconnect. + // We reconnect it by assigning the result of that dispatch to `opPromise`, + // awaiting `opPromise` in an inner function also named `fetch()` and + // returning the result from that. + let opPromise = undefined; + // 1. + const result = new Promise((resolve, reject) => { + const prefix = "Failed to execute 'fetch'"; + webidl.requiredArguments(arguments.length, 1, prefix); + // 2. + const requestObject = new Request(input, init); + + if (span) { + updateSpanFromRequest(span, requestObject); + } - if (!requestObject.headers.has("Accept")) { - ArrayPrototypePush(request.headerList, ["Accept", "*/*"]); - } + // 3. + const request = toInnerRequest(requestObject); + // 4. + if (requestObject.signal.aborted) { + reject(abortFetch(request, null, requestObject.signal.reason)); + return; + } + // 7. + let responseObject = null; + // 9. + let locallyAborted = false; + // 10. + function onabort() { + locallyAborted = true; + reject( + abortFetch(request, responseObject, requestObject.signal.reason), + ); + } + requestObject.signal[abortSignal.add](onabort); - if (!requestObject.headers.has("Accept-Language")) { - ArrayPrototypePush(request.headerList, ["Accept-Language", "*"]); - } + if (!requestObject.headers.has("Accept")) { + ArrayPrototypePush(request.headerList, ["Accept", "*/*"]); + } - // 12. - opPromise = PromisePrototypeCatch( - PromisePrototypeThen( - mainFetch(request, false, requestObject.signal), - (response) => { - // 12.1. - if (locallyAborted) return; - // 12.2. - if (response.aborted) { - reject( - abortFetch( - request, - responseObject, - requestObject.signal.reason, - ), - ); - requestObject.signal[abortSignal.remove](onabort); - return; - } - // 12.3. - if (response.type === "error") { - const err = new TypeError( - "Fetch failed: " + (response.error ?? "unknown error"), - ); - reject(err); + if (!requestObject.headers.has("Accept-Language")) { + ArrayPrototypePush(request.headerList, ["Accept-Language", "*"]); + } + + // 12. + opPromise = PromisePrototypeCatch( + PromisePrototypeThen( + mainFetch(request, false, requestObject.signal), + (response) => { + // 12.1. + if (locallyAborted) return; + // 12.2. + if (response.aborted) { + reject( + abortFetch( + request, + responseObject, + requestObject.signal.reason, + ), + ); + requestObject.signal[abortSignal.remove](onabort); + return; + } + // 12.3. + if (response.type === "error") { + const err = new TypeError( + "Fetch failed: " + (response.error ?? "unknown error"), + ); + reject(err); + requestObject.signal[abortSignal.remove](onabort); + return; + } + responseObject = fromInnerResponse(response, "immutable"); + + if (span) { + updateSpanFromResponse(span, responseObject); + } + + resolve(responseObject); requestObject.signal[abortSignal.remove](onabort); - return; - } - responseObject = fromInnerResponse(response, "immutable"); - resolve(responseObject); + }, + ), + (err) => { + reject(err); requestObject.signal[abortSignal.remove](onabort); }, - ), - (err) => { - reject(err); - requestObject.signal[abortSignal.remove](onabort); - }, - ); - }); - if (opPromise) { - PromisePrototypeCatch(result, () => {}); - return (async function fetch() { - await opPromise; - return result; - })(); + ); + }); + + if (opPromise) { + PromisePrototypeCatch(result, () => {}); + return (async function fetch() { + try { + await opPromise; + return result; + } finally { + if (span) { + endSpan(span); + } + } + })(); + } + // We need to end the span when the promise settles. + // WPT has a test that aborted fetch is settled in the same tick. + // This means we cannot wrap the promise if it is already settled. + // But this is OK, because we can just immediately end the span + // in that case. + if (span) { + // XXX: This should always be true, otherwise `opPromise` would be present. + if (op_fetch_promise_is_settled(result)) { + // It's already settled. + endSpan(span); + } else { + // Not settled yet, we can return a new wrapper promise. + return SafePromisePrototypeFinally(result, () => { + endSpan(span); + }); + } + } + return result; + } finally { + if (span) { + exitSpan(span); + } } - return result; } function abortFetch(request, responseObject, error) { @@ -410,6 +499,19 @@ function abortFetch(request, responseObject, error) { return error; } +/** + * Checks if the given string is a subdomain of the given domain. + * + * @param {String} subdomain + * @param {String} domain + * @returns {Boolean} + */ +function isSubdomain(subdomain, domain) { + const dot = subdomain.length - domain.length - 1; + return dot > 0 && subdomain[dot] === "." && + StringPrototypeEndsWith(subdomain, domain); +} + /** * Handle the Response argument to the WebAssembly streaming APIs, after * resolving if it was passed as a promise. This function should be registered diff --git a/ext/fetch/Cargo.toml b/ext/fetch/Cargo.toml index 2ec1c25f0c6995..96d6a45a1ff1bc 100644 --- a/ext/fetch/Cargo.toml +++ b/ext/fetch/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_fetch" -version = "0.192.0" +version = "0.204.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -22,6 +22,7 @@ deno_permissions.workspace = true deno_tls.workspace = true dyn-clone = "1" error_reporter = "1" +hickory-resolver.workspace = true http.workspace = true http-body-util.workspace = true hyper.workspace = true @@ -32,6 +33,7 @@ percent-encoding.workspace = true rustls-webpki.workspace = true serde.workspace = true serde_json.workspace = true +thiserror.workspace = true tokio.workspace = true tokio-rustls.workspace = true tokio-socks.workspace = true diff --git a/ext/fetch/dns.rs b/ext/fetch/dns.rs new file mode 100644 index 00000000000000..9e21a4c3422927 --- /dev/null +++ b/ext/fetch/dns.rs @@ -0,0 +1,116 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::future::Future; +use std::io; +use std::net::SocketAddr; +use std::pin::Pin; +use std::task::Poll; +use std::task::{self}; +use std::vec; + +use hickory_resolver::error::ResolveError; +use hickory_resolver::name_server::GenericConnector; +use hickory_resolver::name_server::TokioRuntimeProvider; +use hickory_resolver::AsyncResolver; +use hyper_util::client::legacy::connect::dns::GaiResolver; +use hyper_util::client::legacy::connect::dns::Name; +use tokio::task::JoinHandle; +use tower::Service; + +#[derive(Clone, Debug)] +pub enum Resolver { + /// A resolver using blocking `getaddrinfo` calls in a threadpool. + Gai(GaiResolver), + /// hickory-resolver's userspace resolver. + Hickory(AsyncResolver>), +} + +impl Default for Resolver { + fn default() -> Self { + Self::gai() + } +} + +impl Resolver { + pub fn gai() -> Self { + Self::Gai(GaiResolver::new()) + } + + /// Create a [`AsyncResolver`] from system conf. + pub fn hickory() -> Result { + Ok(Self::Hickory( + hickory_resolver::AsyncResolver::tokio_from_system_conf()?, + )) + } + + pub fn hickory_from_async_resolver( + resolver: AsyncResolver>, + ) -> Self { + Self::Hickory(resolver) + } +} + +type SocketAddrs = vec::IntoIter; + +pub struct ResolveFut { + inner: JoinHandle>, +} + +impl Future for ResolveFut { + type Output = Result; + + fn poll( + mut self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + ) -> Poll { + Pin::new(&mut self.inner).poll(cx).map(|res| match res { + Ok(Ok(addrs)) => Ok(addrs), + Ok(Err(e)) => Err(e), + Err(join_err) => { + if join_err.is_cancelled() { + Err(io::Error::new(io::ErrorKind::Interrupted, join_err)) + } else { + Err(io::Error::new(io::ErrorKind::Other, join_err)) + } + } + }) + } +} + +impl Service for Resolver { + type Response = SocketAddrs; + type Error = io::Error; + type Future = ResolveFut; + + fn poll_ready( + &mut self, + _cx: &mut task::Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, name: Name) -> Self::Future { + let task = match self { + Resolver::Gai(gai_resolver) => { + let mut resolver = gai_resolver.clone(); + tokio::spawn(async move { + let result = resolver.call(name).await?; + let x: Vec<_> = result.into_iter().collect(); + let iter: SocketAddrs = x.into_iter(); + Ok(iter) + }) + } + Resolver::Hickory(async_resolver) => { + let resolver = async_resolver.clone(); + tokio::spawn(async move { + let result = resolver.lookup_ip(name.as_str()).await?; + + let x: Vec<_> = + result.into_iter().map(|x| SocketAddr::new(x, 0)).collect(); + let iter: SocketAddrs = x.into_iter(); + Ok(iter) + }) + } + }; + ResolveFut { inner: task } + } +} diff --git a/ext/fetch/fs_fetch_handler.rs b/ext/fetch/fs_fetch_handler.rs index 4c2b81f35645ec..c236dd9c67127d 100644 --- a/ext/fetch/fs_fetch_handler.rs +++ b/ext/fetch/fs_fetch_handler.rs @@ -4,7 +4,6 @@ use crate::CancelHandle; use crate::CancelableResponseFuture; use crate::FetchHandler; -use deno_core::error::type_error; use deno_core::futures::FutureExt; use deno_core::futures::TryFutureExt; use deno_core::futures::TryStreamExt; @@ -42,9 +41,7 @@ impl FetchHandler for FsFetchHandler { .map_err(|_| ())?; Ok::<_, ()>(response) } - .map_err(move |_| { - type_error("NetworkError when attempting to fetch resource") - }) + .map_err(move |_| super::FetchError::NetworkError) .or_cancel(&cancel_handle) .boxed_local(); diff --git a/ext/fetch/lib.deno_fetch.d.ts b/ext/fetch/lib.deno_fetch.d.ts index d219a38592f5d6..8614dec899da6b 100644 --- a/ext/fetch/lib.deno_fetch.d.ts +++ b/ext/fetch/lib.deno_fetch.d.ts @@ -163,6 +163,8 @@ type BodyInit = | FormData | URLSearchParams | ReadableStream + | Iterable + | AsyncIterable | string; /** @category Fetch */ type RequestDestination = diff --git a/ext/fetch/lib.rs b/ext/fetch/lib.rs index 88f3038528cfdc..7a525053b3e0de 100644 --- a/ext/fetch/lib.rs +++ b/ext/fetch/lib.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +pub mod dns; mod fs_fetch_handler; mod proxy; #[cfg(test)] @@ -17,10 +18,6 @@ use std::sync::Arc; use std::task::Context; use std::task::Poll; -use deno_core::anyhow::anyhow; -use deno_core::anyhow::Error; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::futures::stream::Peekable; use deno_core::futures::Future; use deno_core::futures::FutureExt; @@ -28,7 +25,9 @@ use deno_core::futures::Stream; use deno_core::futures::StreamExt; use deno_core::futures::TryFutureExt; use deno_core::op2; +use deno_core::url; use deno_core::url::Url; +use deno_core::v8; use deno_core::AsyncRefCell; use deno_core::AsyncResult; use deno_core::BufView; @@ -42,6 +41,7 @@ use deno_core::OpState; use deno_core::RcRef; use deno_core::Resource; use deno_core::ResourceId; +use deno_permissions::PermissionCheckError; use deno_tls::rustls::RootCertStore; use deno_tls::Proxy; use deno_tls::RootCertStoreProvider; @@ -68,6 +68,7 @@ use http_body_util::BodyExt; use hyper::body::Frame; use hyper_util::client::legacy::connect::HttpConnector; use hyper_util::client::legacy::connect::HttpInfo; +use hyper_util::client::legacy::Builder as HyperClientBuilder; use hyper_util::rt::TokioExecutor; use hyper_util::rt::TokioTimer; use serde::Deserialize; @@ -86,16 +87,30 @@ pub struct Options { pub user_agent: String, pub root_cert_store_provider: Option>, pub proxy: Option, + /// A callback to customize HTTP client configuration. + /// + /// The settings applied with this hook may be overridden by the options + /// provided through `Deno.createHttpClient()` API. For instance, if the hook + /// calls [`hyper_util::client::legacy::Builder::pool_max_idle_per_host`] with + /// a value of 99, and a user calls `Deno.createHttpClient({ poolMaxIdlePerHost: 42 })`, + /// the value that will take effect is 42. + /// + /// For more info on what can be configured, see [`hyper_util::client::legacy::Builder`]. + pub client_builder_hook: Option HyperClientBuilder>, #[allow(clippy::type_complexity)] - pub request_builder_hook: - Option) -> Result<(), AnyError>>, + pub request_builder_hook: Option< + fn(&mut http::Request) -> Result<(), deno_core::error::AnyError>, + >, pub unsafely_ignore_certificate_errors: Option>, pub client_cert_chain_and_key: TlsKeys, pub file_fetch_handler: Rc, + pub resolver: dns::Resolver, } impl Options { - pub fn root_cert_store(&self) -> Result, AnyError> { + pub fn root_cert_store( + &self, + ) -> Result, deno_core::error::AnyError> { Ok(match &self.root_cert_store_provider { Some(provider) => Some(provider.get_or_try_init()?.clone()), None => None, @@ -109,10 +124,12 @@ impl Default for Options { user_agent: "".to_string(), root_cert_store_provider: None, proxy: None, + client_builder_hook: None, request_builder_hook: None, unsafely_ignore_certificate_errors: None, client_cert_chain_and_key: TlsKeys::Null, file_fetch_handler: Rc::new(DefaultFileFetchHandler), + resolver: dns::Resolver::default(), } } } @@ -125,6 +142,7 @@ deno_core::extension!(deno_fetch, op_fetch_send, op_utf8_to_byte_string, op_fetch_custom_client, + op_fetch_promise_is_settled, ], esm = [ "20_headers.js", @@ -144,6 +162,51 @@ deno_core::extension!(deno_fetch, }, ); +#[derive(Debug, thiserror::Error)] +pub enum FetchError { + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error(transparent)] + Permission(#[from] PermissionCheckError), + #[error("NetworkError when attempting to fetch resource")] + NetworkError, + #[error("Fetching files only supports the GET method: received {0}")] + FsNotGet(Method), + #[error("Invalid URL {0}")] + InvalidUrl(Url), + #[error(transparent)] + InvalidHeaderName(#[from] http::header::InvalidHeaderName), + #[error(transparent)] + InvalidHeaderValue(#[from] http::header::InvalidHeaderValue), + #[error("{0:?}")] + DataUrl(data_url::DataUrlError), + #[error("{0:?}")] + Base64(data_url::forgiving_base64::InvalidBase64), + #[error("Blob for the given URL not found.")] + BlobNotFound, + #[error("Url scheme '{0}' not supported")] + SchemeNotSupported(String), + #[error("Request was cancelled")] + RequestCanceled, + #[error(transparent)] + Http(#[from] http::Error), + #[error(transparent)] + ClientCreate(#[from] HttpClientCreateError), + #[error(transparent)] + Url(#[from] url::ParseError), + #[error(transparent)] + Method(#[from] http::method::InvalidMethod), + #[error(transparent)] + ClientSend(#[from] ClientSendError), + #[error(transparent)] + RequestBuilderHook(deno_core::error::AnyError), + #[error(transparent)] + Io(#[from] std::io::Error), + // Only used for node upgrade + #[error(transparent)] + Hyper(#[from] hyper::Error), +} + pub type CancelableResponseFuture = Pin>>; @@ -170,11 +233,7 @@ impl FetchHandler for DefaultFileFetchHandler { _state: &mut OpState, _url: &Url, ) -> (CancelableResponseFuture, Option>) { - let fut = async move { - Ok(Err(type_error( - "NetworkError when attempting to fetch resource", - ))) - }; + let fut = async move { Ok(Err(FetchError::NetworkError)) }; (Box::pin(fut), None) } } @@ -191,7 +250,7 @@ pub struct FetchReturn { pub fn get_or_create_client_from_state( state: &mut OpState, -) -> Result { +) -> Result { if let Some(client) = state.try_borrow::() { Ok(client.clone()) } else { @@ -204,13 +263,16 @@ pub fn get_or_create_client_from_state( pub fn create_client_from_options( options: &Options, -) -> Result { +) -> Result { create_http_client( &options.user_agent, CreateHttpClientOptions { - root_cert_store: options.root_cert_store()?, + root_cert_store: options + .root_cert_store() + .map_err(HttpClientCreateError::RootCertStore)?, ca_certs: vec![], proxy: options.proxy.clone(), + dns_resolver: options.resolver.clone(), unsafely_ignore_certificate_errors: options .unsafely_ignore_certificate_errors .clone(), @@ -223,6 +285,7 @@ pub fn create_client_from_options( pool_idle_timeout: None, http1: true, http2: true, + client_builder_hook: options.client_builder_hook, }, ) } @@ -230,7 +293,9 @@ pub fn create_client_from_options( #[allow(clippy::type_complexity)] pub struct ResourceToBodyAdapter( Rc, - Option>>>>, + Option< + Pin>>>, + >, ); impl ResourceToBodyAdapter { @@ -246,7 +311,7 @@ unsafe impl Send for ResourceToBodyAdapter {} unsafe impl Sync for ResourceToBodyAdapter {} impl Stream for ResourceToBodyAdapter { - type Item = Result; + type Item = Result; fn poll_next( self: Pin<&mut Self>, @@ -276,7 +341,7 @@ impl Stream for ResourceToBodyAdapter { impl hyper::body::Body for ResourceToBodyAdapter { type Data = Bytes; - type Error = Error; + type Error = deno_core::error::AnyError; fn poll_frame( self: Pin<&mut Self>, @@ -301,13 +366,13 @@ pub trait FetchPermissions { &mut self, url: &Url, api_name: &str, - ) -> Result<(), AnyError>; + ) -> Result<(), PermissionCheckError>; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] fn check_read<'a>( &mut self, p: &'a Path, api_name: &str, - ) -> Result, AnyError>; + ) -> Result, PermissionCheckError>; } impl FetchPermissions for deno_permissions::PermissionsContainer { @@ -316,7 +381,7 @@ impl FetchPermissions for deno_permissions::PermissionsContainer { &mut self, url: &Url, api_name: &str, - ) -> Result<(), AnyError> { + ) -> Result<(), PermissionCheckError> { deno_permissions::PermissionsContainer::check_net_url(self, url, api_name) } @@ -325,7 +390,7 @@ impl FetchPermissions for deno_permissions::PermissionsContainer { &mut self, path: &'a Path, api_name: &str, - ) -> Result, AnyError> { + ) -> Result, PermissionCheckError> { deno_permissions::PermissionsContainer::check_read_path( self, path, @@ -334,7 +399,7 @@ impl FetchPermissions for deno_permissions::PermissionsContainer { } } -#[op2] +#[op2(stack_trace)] #[serde] #[allow(clippy::too_many_arguments)] pub fn op_fetch( @@ -346,12 +411,15 @@ pub fn op_fetch( has_body: bool, #[buffer] data: Option, #[smi] resource: Option, -) -> Result +) -> Result where FP: FetchPermissions + 'static, { let (client, allow_host) = if let Some(rid) = client_rid { - let r = state.resource_table.get::(rid)?; + let r = state + .resource_table + .get::(rid) + .map_err(FetchError::Resource)?; (r.client.clone(), r.allow_host) } else { (get_or_create_client_from_state(state)?, false) @@ -364,9 +432,7 @@ where let scheme = url.scheme(); let (request_rid, cancel_handle_rid) = match scheme { "file" => { - let path = url.to_file_path().map_err(|_| { - type_error("NetworkError when attempting to fetch resource") - })?; + let path = url.to_file_path().map_err(|_| FetchError::NetworkError)?; let permissions = state.borrow_mut::(); let path = permissions.check_read(&path, "fetch()")?; let url = match path { @@ -375,9 +441,7 @@ where }; if method != Method::GET { - return Err(type_error(format!( - "Fetching files only supports the GET method: received {method}" - ))); + return Err(FetchError::FsNotGet(method)); } let Options { @@ -402,7 +466,7 @@ where let uri = url .as_str() .parse::() - .map_err(|_| type_error(format!("Invalid URL {url}")))?; + .map_err(|_| FetchError::InvalidUrl(url.clone()))?; let mut con_len = None; let body = if has_body { @@ -416,7 +480,10 @@ where .boxed() } (_, Some(resource)) => { - let resource = state.resource_table.take_any(resource)?; + let resource = state + .resource_table + .take_any(resource) + .map_err(FetchError::Resource)?; match resource.size_hint() { (body_size, Some(n)) if body_size == n && body_size > 0 => { con_len = Some(body_size); @@ -453,10 +520,8 @@ where } for (key, value) in headers { - let name = HeaderName::from_bytes(&key) - .map_err(|err| type_error(err.to_string()))?; - let v = HeaderValue::from_bytes(&value) - .map_err(|err| type_error(err.to_string()))?; + let name = HeaderName::from_bytes(&key)?; + let v = HeaderValue::from_bytes(&value)?; if (name != HOST || allow_host) && name != CONTENT_LENGTH { request.headers_mut().append(name, v); @@ -474,20 +539,18 @@ where let options = state.borrow::(); if let Some(request_builder_hook) = options.request_builder_hook { request_builder_hook(&mut request) - .map_err(|err| type_error(err.to_string()))?; + .map_err(FetchError::RequestBuilderHook)?; } let cancel_handle = CancelHandle::new_rc(); let cancel_handle_ = cancel_handle.clone(); - let fut = { - async move { - client - .send(request) - .map_err(Into::into) - .or_cancel(cancel_handle_) - .await - } + let fut = async move { + client + .send(request) + .map_err(Into::into) + .or_cancel(cancel_handle_) + .await }; let request_rid = state.resource_table.add(FetchRequestResource { @@ -501,12 +564,10 @@ where (request_rid, Some(cancel_handle_rid)) } "data" => { - let data_url = DataUrl::process(url.as_str()) - .map_err(|e| type_error(format!("{e:?}")))?; + let data_url = + DataUrl::process(url.as_str()).map_err(FetchError::DataUrl)?; - let (body, _) = data_url - .decode_to_vec() - .map_err(|e| type_error(format!("{e:?}")))?; + let (body, _) = data_url.decode_to_vec().map_err(FetchError::Base64)?; let body = http_body_util::Full::new(body.into()) .map_err(|never| match never {}) .boxed(); @@ -528,11 +589,9 @@ where "blob" => { // Blob URL resolution happens in the JS side of fetch. If we got here is // because the URL isn't an object URL. - return Err(type_error("Blob for the given URL not found.")); - } - _ => { - return Err(type_error(format!("Url scheme '{scheme}' not supported"))) + return Err(FetchError::BlobNotFound); } + _ => return Err(FetchError::SchemeNotSupported(scheme.to_string())), }; Ok(FetchReturn { @@ -564,11 +623,12 @@ pub struct FetchResponse { pub async fn op_fetch_send( state: Rc>, #[smi] rid: ResourceId, -) -> Result { +) -> Result { let request = state .borrow_mut() .resource_table - .take::(rid)?; + .take::(rid) + .map_err(FetchError::Resource)?; let request = Rc::try_unwrap(request) .ok() @@ -581,22 +641,23 @@ pub async fn op_fetch_send( // If any error in the chain is a hyper body error, return that as a special result we can use to // reconstruct an error chain (eg: `new TypeError(..., { cause: new Error(...) })`). // TODO(mmastrac): it would be a lot easier if we just passed a v8::Global through here instead - let mut err_ref: &dyn std::error::Error = err.as_ref(); - while let Some(err_src) = std::error::Error::source(err_ref) { - if let Some(err_src) = err_src.downcast_ref::() { - if let Some(err_src) = std::error::Error::source(err_src) { - return Ok(FetchResponse { - error: Some((err.to_string(), err_src.to_string())), - ..Default::default() - }); + + if let FetchError::ClientSend(err_src) = &err { + if let Some(client_err) = std::error::Error::source(&err_src.source) { + if let Some(err_src) = client_err.downcast_ref::() { + if let Some(err_src) = std::error::Error::source(err_src) { + return Ok(FetchResponse { + error: Some((err.to_string(), err_src.to_string())), + ..Default::default() + }); + } } } - err_ref = err_src; } - return Err(type_error(err.to_string())); + return Err(err); } - Err(_) => return Err(type_error("Request was cancelled")), + Err(_) => return Err(FetchError::RequestCanceled), }; let status = res.status(); @@ -636,7 +697,7 @@ pub async fn op_fetch_send( } type CancelableResponseResult = - Result, AnyError>, Canceled>; + Result, FetchError>, Canceled>; pub struct FetchRequestResource { pub future: Pin>>, @@ -691,7 +752,7 @@ impl FetchResponseResource { } } - pub async fn upgrade(self) -> Result { + pub async fn upgrade(self) -> Result { let reader = self.response_reader.into_inner(); match reader { FetchResponseReader::Start(resp) => Ok(hyper::upgrade::on(resp).await?), @@ -746,7 +807,9 @@ impl Resource for FetchResponseResource { // safely call `await` on it without creating a race condition. Some(_) => match reader.as_mut().next().await.unwrap() { Ok(chunk) => assert!(chunk.is_empty()), - Err(err) => break Err(type_error(err.to_string())), + Err(err) => { + break Err(deno_core::error::type_error(err.to_string())) + } }, None => break Ok(BufView::empty()), } @@ -791,6 +854,8 @@ pub struct CreateHttpClientArgs { proxy: Option, pool_max_idle_per_host: Option, pool_idle_timeout: Option, + #[serde(default)] + use_hickory_resolver: bool, #[serde(default = "default_true")] http1: bool, #[serde(default = "default_true")] @@ -803,13 +868,13 @@ fn default_true() -> bool { true } -#[op2] +#[op2(stack_trace)] #[smi] pub fn op_fetch_custom_client( state: &mut OpState, #[serde] args: CreateHttpClientArgs, #[cppgc] tls_keys: &TlsKeysHolder, -) -> Result +) -> Result where FP: FetchPermissions + 'static, { @@ -829,9 +894,18 @@ where let client = create_http_client( &options.user_agent, CreateHttpClientOptions { - root_cert_store: options.root_cert_store()?, + root_cert_store: options + .root_cert_store() + .map_err(HttpClientCreateError::RootCertStore)?, ca_certs, proxy: args.proxy, + dns_resolver: if args.use_hickory_resolver { + dns::Resolver::hickory() + .map_err(deno_core::error::AnyError::new) + .map_err(FetchError::Resource)? + } else { + dns::Resolver::default() + }, unsafely_ignore_certificate_errors: options .unsafely_ignore_certificate_errors .clone(), @@ -849,6 +923,7 @@ where ), http1: args.http1, http2: args.http2, + client_builder_hook: options.client_builder_hook, }, )?; @@ -863,12 +938,14 @@ pub struct CreateHttpClientOptions { pub root_cert_store: Option, pub ca_certs: Vec>, pub proxy: Option, + pub dns_resolver: dns::Resolver, pub unsafely_ignore_certificate_errors: Option>, pub client_cert_chain_and_key: Option, pub pool_max_idle_per_host: Option, pub pool_idle_timeout: Option>, pub http1: bool, pub http2: bool, + pub client_builder_hook: Option HyperClientBuilder>, } impl Default for CreateHttpClientOptions { @@ -877,29 +954,46 @@ impl Default for CreateHttpClientOptions { root_cert_store: None, ca_certs: vec![], proxy: None, + dns_resolver: dns::Resolver::default(), unsafely_ignore_certificate_errors: None, client_cert_chain_and_key: None, pool_max_idle_per_host: None, pool_idle_timeout: None, http1: true, http2: true, + client_builder_hook: None, } } } +#[derive(Debug, thiserror::Error)] +pub enum HttpClientCreateError { + #[error(transparent)] + Tls(deno_tls::TlsError), + #[error("Illegal characters in User-Agent: received {0}")] + InvalidUserAgent(String), + #[error("invalid proxy url")] + InvalidProxyUrl, + #[error("Cannot create Http Client: either `http1` or `http2` needs to be set to true")] + HttpVersionSelectionInvalid, + #[error(transparent)] + RootCertStore(deno_core::error::AnyError), +} + /// Create new instance of async Client. This client supports /// proxies and doesn't follow redirects. pub fn create_http_client( user_agent: &str, options: CreateHttpClientOptions, -) -> Result { +) -> Result { let mut tls_config = deno_tls::create_client_config( options.root_cert_store, options.ca_certs, options.unsafely_ignore_certificate_errors, options.client_cert_chain_and_key.into(), deno_tls::SocketUse::Http, - )?; + ) + .map_err(HttpClientCreateError::Tls)?; // Proxy TLS should not send ALPN tls_config.alpn_protocols.clear(); @@ -915,24 +1009,26 @@ pub fn create_http_client( tls_config.alpn_protocols = alpn_protocols; let tls_config = Arc::from(tls_config); - let mut http_connector = HttpConnector::new(); + let mut http_connector = + HttpConnector::new_with_resolver(options.dns_resolver.clone()); http_connector.enforce_http(false); let user_agent = user_agent.parse::().map_err(|_| { - type_error(format!( - "Illegal characters in User-Agent: received {user_agent}" - )) + HttpClientCreateError::InvalidUserAgent(user_agent.to_string()) })?; - let mut builder = - hyper_util::client::legacy::Builder::new(TokioExecutor::new()); + let mut builder = HyperClientBuilder::new(TokioExecutor::new()); builder.timer(TokioTimer::new()); builder.pool_timer(TokioTimer::new()); + if let Some(client_builder_hook) = options.client_builder_hook { + builder = client_builder_hook(builder); + } + let mut proxies = proxy::from_env(); if let Some(proxy) = options.proxy { let mut intercept = proxy::Intercept::all(&proxy.url) - .ok_or_else(|| type_error("invalid proxy url"))?; + .ok_or_else(|| HttpClientCreateError::InvalidProxyUrl)?; if let Some(basic_auth) = &proxy.basic_auth { intercept.set_auth(&basic_auth.username, &basic_auth.password); } @@ -964,7 +1060,7 @@ pub fn create_http_client( } (true, true) => {} (false, false) => { - return Err(type_error("Cannot create Http Client: either `http1` or `http2` needs to be set to true")) + return Err(HttpClientCreateError::HttpVersionSelectionInvalid) } } @@ -980,10 +1076,8 @@ pub fn create_http_client( #[op2] #[serde] -pub fn op_utf8_to_byte_string( - #[string] input: String, -) -> Result { - Ok(input.into()) +pub fn op_utf8_to_byte_string(#[string] input: String) -> ByteString { + input.into() } #[derive(Clone, Debug)] @@ -994,7 +1088,7 @@ pub struct Client { user_agent: HeaderValue, } -type Connector = proxy::ProxyConnector; +type Connector = proxy::ProxyConnector>; // clippy is wrong here #[allow(clippy::declare_interior_mutable_const)] @@ -1003,7 +1097,7 @@ const STAR_STAR: HeaderValue = HeaderValue::from_static("*/*"); #[derive(Debug)] pub struct ClientSendError { uri: Uri, - source: hyper_util::client::legacy::Error, + pub source: hyper_util::client::legacy::Error, } impl ClientSendError { @@ -1075,12 +1169,14 @@ impl Client { .oneshot(req) .await .map_err(|e| ClientSendError { uri, source: e })?; - Ok(resp.map(|b| b.map_err(|e| anyhow!(e)).boxed())) + Ok(resp.map(|b| b.map_err(|e| deno_core::anyhow::anyhow!(e)).boxed())) } } -pub type ReqBody = http_body_util::combinators::BoxBody; -pub type ResBody = http_body_util::combinators::BoxBody; +pub type ReqBody = + http_body_util::combinators::BoxBody; +pub type ResBody = + http_body_util::combinators::BoxBody; /// Copied from https://github.com/seanmonstar/reqwest/blob/b9d62a0323d96f11672a61a17bf8849baec00275/src/async_impl/request.rs#L572 /// Check the request URL for a "username:password" type authority, and if @@ -1112,3 +1208,8 @@ pub fn extract_authority(url: &mut Url) -> Option<(String, Option)> { None } + +#[op2(fast)] +fn op_fetch_promise_is_settled(promise: v8::Local) -> bool { + promise.state() != v8::PromiseState::Pending +} diff --git a/ext/fetch/tests.rs b/ext/fetch/tests.rs index dad1b34a9e8016..e053c6b1cfec5f 100644 --- a/ext/fetch/tests.rs +++ b/ext/fetch/tests.rs @@ -1,6 +1,8 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use std::net::SocketAddr; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering::SeqCst; use std::sync::Arc; use bytes::Bytes; @@ -10,6 +12,8 @@ use http_body_util::BodyExt; use tokio::io::AsyncReadExt; use tokio::io::AsyncWriteExt; +use crate::dns; + use super::create_http_client; use super::CreateHttpClientOptions; @@ -17,6 +21,53 @@ static EXAMPLE_CRT: &[u8] = include_bytes!("../tls/testdata/example1_cert.der"); static EXAMPLE_KEY: &[u8] = include_bytes!("../tls/testdata/example1_prikey.der"); +#[test] +fn test_userspace_resolver() { + let thread_counter = Arc::new(AtomicUsize::new(0)); + + let thread_counter_ref = thread_counter.clone(); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .on_thread_start(move || { + thread_counter_ref.fetch_add(1, SeqCst); + }) + .build() + .unwrap(); + + rt.block_on(async move { + assert_eq!(thread_counter.load(SeqCst), 0); + let src_addr = create_https_server(true).await; + assert_eq!(src_addr.ip().to_string(), "127.0.0.1"); + // use `localhost` to ensure dns step happens. + let addr = format!("localhost:{}", src_addr.port()); + + let hickory = hickory_resolver::AsyncResolver::tokio( + Default::default(), + Default::default(), + ); + + assert_eq!(thread_counter.load(SeqCst), 0); + rust_test_client_with_resolver( + None, + addr.clone(), + "https", + http::Version::HTTP_2, + dns::Resolver::hickory_from_async_resolver(hickory), + ) + .await; + assert_eq!(thread_counter.load(SeqCst), 0, "userspace resolver shouldn't spawn new threads."); + rust_test_client_with_resolver( + None, + addr.clone(), + "https", + http::Version::HTTP_2, + dns::Resolver::gai(), + ) + .await; + assert_eq!(thread_counter.load(SeqCst), 1, "getaddrinfo is called inside spawn_blocking, so tokio spawn a new worker thread for it."); + }); +} + #[tokio::test] async fn test_https_proxy_http11() { let src_addr = create_https_server(false).await; @@ -52,27 +103,30 @@ async fn test_socks_proxy_h2() { run_test_client(prx_addr, src_addr, "socks5", http::Version::HTTP_2).await; } -async fn run_test_client( - prx_addr: SocketAddr, - src_addr: SocketAddr, +async fn rust_test_client_with_resolver( + prx_addr: Option, + src_addr: String, proto: &str, ver: http::Version, + resolver: dns::Resolver, ) { let client = create_http_client( "fetch/test", CreateHttpClientOptions { root_cert_store: None, ca_certs: vec![], - proxy: Some(deno_tls::Proxy { - url: format!("{}://{}", proto, prx_addr), + proxy: prx_addr.map(|p| deno_tls::Proxy { + url: format!("{}://{}", proto, p), basic_auth: None, }), unsafely_ignore_certificate_errors: Some(vec![]), client_cert_chain_and_key: None, pool_max_idle_per_host: None, pool_idle_timeout: None, + dns_resolver: resolver, http1: true, http2: true, + client_builder_hook: None, }, ) .unwrap(); @@ -92,6 +146,22 @@ async fn run_test_client( assert_eq!(hello, "hello from server"); } +async fn run_test_client( + prx_addr: SocketAddr, + src_addr: SocketAddr, + proto: &str, + ver: http::Version, +) { + rust_test_client_with_resolver( + Some(prx_addr), + src_addr.to_string(), + proto, + ver, + Default::default(), + ) + .await +} + async fn create_https_server(allow_h2: bool) -> SocketAddr { let mut tls_config = deno_tls::rustls::server::ServerConfig::builder() .with_no_client_auth() diff --git a/ext/ffi/Cargo.toml b/ext/ffi/Cargo.toml index 8c4ccdc9babb6c..23deb308102b11 100644 --- a/ext/ffi/Cargo.toml +++ b/ext/ffi/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_ffi" -version = "0.155.0" +version = "0.167.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -21,9 +21,12 @@ dynasmrt = "1.2.3" libffi = "=3.2.0" libffi-sys = "=2.3.0" log.workspace = true +num-bigint.workspace = true serde.workspace = true serde-value = "0.7" serde_json = "1.0" +thiserror.workspace = true +tokio.workspace = true [target.'cfg(windows)'.dependencies] winapi = { workspace = true, features = ["errhandlingapi", "minwindef", "ntdef", "winbase", "winnt"] } diff --git a/ext/ffi/call.rs b/ext/ffi/call.rs index 3572b9e813e430..c964071a09f538 100644 --- a/ext/ffi/call.rs +++ b/ext/ffi/call.rs @@ -7,23 +7,38 @@ use crate::symbol::NativeType; use crate::symbol::Symbol; use crate::FfiPermissions; use crate::ForeignFunction; -use deno_core::anyhow::anyhow; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::serde_json::Value; +use deno_core::serde_v8::BigInt as V8BigInt; use deno_core::serde_v8::ExternalPointer; use deno_core::unsync::spawn_blocking; use deno_core::v8; use deno_core::OpState; use deno_core::ResourceId; use libffi::middle::Arg; +use num_bigint::BigInt; use serde::Serialize; use std::cell::RefCell; use std::ffi::c_void; use std::future::Future; use std::rc::Rc; +#[derive(Debug, thiserror::Error)] +pub enum CallError { + #[error(transparent)] + IR(#[from] IRError), + #[error("Nonblocking FFI call failed: {0}")] + NonblockingCallFailure(#[source] tokio::task::JoinError), + #[error("Invalid FFI symbol name: '{0}'")] + InvalidSymbol(String), + #[error(transparent)] + Permission(#[from] deno_permissions::PermissionCheckError), + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error(transparent)] + Callback(#[from] super::CallbackError), +} + // SAFETY: Makes an FFI call unsafe fn ffi_call_rtype_struct( cif: &libffi::middle::Cif, @@ -45,7 +60,7 @@ pub(crate) fn ffi_call_sync<'scope>( args: v8::FunctionCallbackArguments, symbol: &Symbol, out_buffer: Option, -) -> Result +) -> Result where 'scope: 'scope, { @@ -191,6 +206,7 @@ where #[serde(untagged)] pub enum FfiValue { Value(Value), + BigInt(V8BigInt), External(ExternalPointer), } @@ -201,7 +217,7 @@ fn ffi_call( parameter_types: &[NativeType], result_type: NativeType, out_buffer: Option, -) -> Result { +) -> FfiValue { let call_args: Vec = call_args .iter() .enumerate() @@ -214,7 +230,7 @@ fn ffi_call( // SAFETY: types in the `Cif` match the actual calling convention and // types of symbol. unsafe { - Ok(match result_type { + match result_type { NativeType::Void => { cif.call::<()>(fun_ptr, &call_args); FfiValue::Value(Value::from(())) @@ -240,18 +256,18 @@ fn ffi_call( NativeType::I32 => { FfiValue::Value(Value::from(cif.call::(fun_ptr, &call_args))) } - NativeType::U64 => { - FfiValue::Value(Value::from(cif.call::(fun_ptr, &call_args))) - } - NativeType::I64 => { - FfiValue::Value(Value::from(cif.call::(fun_ptr, &call_args))) - } - NativeType::USize => { - FfiValue::Value(Value::from(cif.call::(fun_ptr, &call_args))) - } - NativeType::ISize => { - FfiValue::Value(Value::from(cif.call::(fun_ptr, &call_args))) - } + NativeType::U64 => FfiValue::BigInt(V8BigInt::from(BigInt::from( + cif.call::(fun_ptr, &call_args), + ))), + NativeType::I64 => FfiValue::BigInt(V8BigInt::from(BigInt::from( + cif.call::(fun_ptr, &call_args), + ))), + NativeType::USize => FfiValue::BigInt(V8BigInt::from(BigInt::from( + cif.call::(fun_ptr, &call_args), + ))), + NativeType::ISize => FfiValue::BigInt(V8BigInt::from(BigInt::from( + cif.call::(fun_ptr, &call_args), + ))), NativeType::F32 => { FfiValue::Value(Value::from(cif.call::(fun_ptr, &call_args))) } @@ -267,11 +283,11 @@ fn ffi_call( ffi_call_rtype_struct(cif, &fun_ptr, call_args, out_buffer.unwrap().0); FfiValue::Value(Value::Null) } - }) + } } } -#[op2(async)] +#[op2(async, stack_trace)] #[serde] pub fn op_ffi_call_ptr_nonblocking( scope: &mut v8::HandleScope, @@ -280,7 +296,7 @@ pub fn op_ffi_call_ptr_nonblocking( #[serde] def: ForeignFunction, parameters: v8::Local, out_buffer: Option>, -) -> Result>, AnyError> +) -> Result>, CallError> where FP: FfiPermissions + 'static, { @@ -309,7 +325,7 @@ where Ok(async move { let result = join_handle .await - .map_err(|err| anyhow!("Nonblocking FFI call failed: {}", err))??; + .map_err(CallError::NonblockingCallFailure)?; // SAFETY: Same return type declared to libffi; trust user to have it right beyond that. Ok(result) }) @@ -325,16 +341,17 @@ pub fn op_ffi_call_nonblocking( #[string] symbol: String, parameters: v8::Local, out_buffer: Option>, -) -> Result>, AnyError> { +) -> Result>, CallError> { let symbol = { let state = state.borrow(); - let resource = state.resource_table.get::(rid)?; + let resource = state + .resource_table + .get::(rid) + .map_err(CallError::Resource)?; let symbols = &resource.symbols; *symbols .get(&symbol) - .ok_or_else(|| { - type_error(format!("Invalid FFI symbol name: '{symbol}'")) - })? + .ok_or_else(|| CallError::InvalidSymbol(symbol))? .clone() }; @@ -362,13 +379,13 @@ pub fn op_ffi_call_nonblocking( Ok(async move { let result = join_handle .await - .map_err(|err| anyhow!("Nonblocking FFI call failed: {}", err))??; + .map_err(CallError::NonblockingCallFailure)?; // SAFETY: Same return type declared to libffi; trust user to have it right beyond that. Ok(result) }) } -#[op2(reentrant)] +#[op2(reentrant, stack_trace)] #[serde] pub fn op_ffi_call_ptr( scope: &mut v8::HandleScope, @@ -377,7 +394,7 @@ pub fn op_ffi_call_ptr( #[serde] def: ForeignFunction, parameters: v8::Local, out_buffer: Option>, -) -> Result +) -> Result where FP: FfiPermissions + 'static, { @@ -399,7 +416,7 @@ where &def.parameters, def.result.clone(), out_buffer_ptr, - )?; + ); // SAFETY: Same return type declared to libffi; trust user to have it right beyond that. Ok(result) } diff --git a/ext/ffi/callback.rs b/ext/ffi/callback.rs index 6fa166f52bcc04..eff14503d1d19b 100644 --- a/ext/ffi/callback.rs +++ b/ext/ffi/callback.rs @@ -3,7 +3,6 @@ use crate::symbol::NativeType; use crate::FfiPermissions; use crate::ForeignFunction; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::v8; use deno_core::v8::TryCatch; @@ -34,6 +33,16 @@ thread_local! { static LOCAL_THREAD_ID: RefCell = const { RefCell::new(0) }; } +#[derive(Debug, thiserror::Error)] +pub enum CallbackError { + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error(transparent)] + Permission(#[from] deno_permissions::PermissionCheckError), + #[error(transparent)] + Other(deno_core::error::AnyError), +} + #[derive(Clone)] pub struct PtrSymbol { pub cif: libffi::middle::Cif, @@ -44,7 +53,7 @@ impl PtrSymbol { pub fn new( fn_ptr: *mut c_void, def: &ForeignFunction, - ) -> Result { + ) -> Result { let ptr = libffi::middle::CodePtr::from_ptr(fn_ptr as _); let cif = libffi::middle::Cif::new( def @@ -52,8 +61,13 @@ impl PtrSymbol { .clone() .into_iter() .map(libffi::middle::Type::try_from) - .collect::, _>>()?, - def.result.clone().try_into()?, + .collect::, _>>() + .map_err(CallbackError::Other)?, + def + .result + .clone() + .try_into() + .map_err(CallbackError::Other)?, ); Ok(Self { cif, ptr }) @@ -522,10 +536,12 @@ unsafe fn do_ffi_callback( pub fn op_ffi_unsafe_callback_ref( state: Rc>, #[smi] rid: ResourceId, -) -> Result>, AnyError> { +) -> Result, CallbackError> { let state = state.borrow(); - let callback_resource = - state.resource_table.get::(rid)?; + let callback_resource = state + .resource_table + .get::(rid) + .map_err(CallbackError::Resource)?; Ok(async move { let info: &mut CallbackInfo = @@ -536,7 +552,6 @@ pub fn op_ffi_unsafe_callback_ref( .into_future() .or_cancel(callback_resource.cancel.clone()) .await; - Ok(()) }) } @@ -546,13 +561,13 @@ pub struct RegisterCallbackArgs { result: NativeType, } -#[op2] +#[op2(stack_trace)] pub fn op_ffi_unsafe_callback_create( state: &mut OpState, scope: &mut v8::HandleScope<'scope>, #[serde] args: RegisterCallbackArgs, cb: v8::Local, -) -> Result, AnyError> +) -> Result, CallbackError> where FP: FfiPermissions + 'static, { @@ -593,8 +608,10 @@ where .parameters .into_iter() .map(libffi::middle::Type::try_from) - .collect::, _>>()?, - libffi::middle::Type::try_from(args.result)?, + .collect::, _>>() + .map_err(CallbackError::Other)?, + libffi::middle::Type::try_from(args.result) + .map_err(CallbackError::Other)?, ); // SAFETY: CallbackInfo is leaked, is not null and stays valid as long as the callback exists. @@ -624,14 +641,16 @@ pub fn op_ffi_unsafe_callback_close( state: &mut OpState, scope: &mut v8::HandleScope, #[smi] rid: ResourceId, -) -> Result<(), AnyError> { +) -> Result<(), CallbackError> { // SAFETY: This drops the closure and the callback info associated with it. // Any retained function pointers to the closure become dangling pointers. // It is up to the user to know that it is safe to call the `close()` on the // UnsafeCallback instance. unsafe { - let callback_resource = - state.resource_table.take::(rid)?; + let callback_resource = state + .resource_table + .take::(rid) + .map_err(CallbackError::Resource)?; let info = Box::from_raw(callback_resource.info); let _ = v8::Global::from_raw(scope, info.callback); let _ = v8::Global::from_raw(scope, info.context); diff --git a/ext/ffi/dlfcn.rs b/ext/ffi/dlfcn.rs index 10199bf858583d..e1bb121d8c39b6 100644 --- a/ext/ffi/dlfcn.rs +++ b/ext/ffi/dlfcn.rs @@ -6,8 +6,6 @@ use crate::symbol::Symbol; use crate::turbocall; use crate::turbocall::Turbocall; use crate::FfiPermissions; -use deno_core::error::generic_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::v8; use deno_core::GarbageCollected; @@ -17,10 +15,27 @@ use dlopen2::raw::Library; use serde::Deserialize; use serde_value::ValueDeserializer; use std::borrow::Cow; +use std::cell::RefCell; use std::collections::HashMap; use std::ffi::c_void; use std::rc::Rc; +#[derive(Debug, thiserror::Error)] +pub enum DlfcnError { + #[error("Failed to register symbol {symbol}: {error}")] + RegisterSymbol { + symbol: String, + #[source] + error: dlopen2::Error, + }, + #[error(transparent)] + Dlopen(#[from] dlopen2::Error), + #[error(transparent)] + Permission(#[from] deno_permissions::PermissionCheckError), + #[error(transparent)] + Other(deno_core::error::AnyError), +} + pub struct DynamicLibraryResource { lib: Library, pub symbols: HashMap>, @@ -37,7 +52,7 @@ impl Resource for DynamicLibraryResource { } impl DynamicLibraryResource { - pub fn get_static(&self, symbol: String) -> Result<*mut c_void, AnyError> { + pub fn get_static(&self, symbol: String) -> Result<*mut c_void, DlfcnError> { // By default, Err returned by this function does not tell // which symbol wasn't exported. So we'll modify the error // message to include the name of symbol. @@ -45,9 +60,7 @@ impl DynamicLibraryResource { // SAFETY: The obtained T symbol is the size of a pointer. match unsafe { self.lib.symbol::<*mut c_void>(&symbol) } { Ok(value) => Ok(Ok(value)), - Err(err) => Err(generic_error(format!( - "Failed to register symbol {symbol}: {err}" - ))), + Err(error) => Err(DlfcnError::RegisterSymbol { symbol, error }), }? } } @@ -111,17 +124,20 @@ pub struct FfiLoadArgs { symbols: HashMap, } -#[op2] +#[op2(stack_trace)] pub fn op_ffi_load<'scope, FP>( scope: &mut v8::HandleScope<'scope>, - state: &mut OpState, + state: Rc>, #[serde] args: FfiLoadArgs, -) -> Result, AnyError> +) -> Result, DlfcnError> where FP: FfiPermissions + 'static, { - let permissions = state.borrow_mut::(); - let path = permissions.check_partial_with_path(&args.path)?; + let path = { + let mut state = state.borrow_mut(); + let permissions = state.borrow_mut::(); + permissions.check_partial_with_path(&args.path)? + }; let lib = Library::open(&path).map_err(|e| { dlopen2::Error::OpeningLibraryError(std::io::Error::new( @@ -152,15 +168,16 @@ where // SAFETY: The obtained T symbol is the size of a pointer. match unsafe { resource.lib.symbol::<*const c_void>(symbol) } { Ok(value) => Ok(value), - Err(err) => if foreign_fn.optional { + Err(error) => if foreign_fn.optional { let null: v8::Local = v8::null(scope).into(); let func_key = v8::String::new(scope, &symbol_key).unwrap(); obj.set(scope, func_key.into(), null); break 'register_symbol; } else { - Err(generic_error(format!( - "Failed to register symbol {symbol}: {err}" - ))) + Err(DlfcnError::RegisterSymbol { + symbol: symbol.to_owned(), + error, + }) }, }?; @@ -171,8 +188,13 @@ where .clone() .into_iter() .map(libffi::middle::Type::try_from) - .collect::, _>>()?, - foreign_fn.result.clone().try_into()?, + .collect::, _>>() + .map_err(DlfcnError::Other)?, + foreign_fn + .result + .clone() + .try_into() + .map_err(DlfcnError::Other)?, ); let func_key = v8::String::new(scope, &symbol_key).unwrap(); @@ -197,6 +219,7 @@ where } } + let mut state = state.borrow_mut(); let out = v8::Array::new(scope, 2); let rid = state.resource_table.add(resource); let rid_v8 = v8::Integer::new_from_unsigned(scope, rid); diff --git a/ext/ffi/ir.rs b/ext/ffi/ir.rs index ebf64945b4aafa..2e80842166fa7e 100644 --- a/ext/ffi/ir.rs +++ b/ext/ffi/ir.rs @@ -1,13 +1,55 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use crate::symbol::NativeType; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::v8; use libffi::middle::Arg; use std::ffi::c_void; use std::ptr; +#[derive(Debug, thiserror::Error)] +pub enum IRError { + #[error("Invalid FFI u8 type, expected boolean")] + InvalidU8ExpectedBoolean, + #[error("Invalid FFI u8 type, expected unsigned integer")] + InvalidU8ExpectedUnsignedInteger, + #[error("Invalid FFI i8 type, expected integer")] + InvalidI8, + #[error("Invalid FFI u16 type, expected unsigned integer")] + InvalidU16, + #[error("Invalid FFI i16 type, expected integer")] + InvalidI16, + #[error("Invalid FFI u32 type, expected unsigned integer")] + InvalidU32, + #[error("Invalid FFI i32 type, expected integer")] + InvalidI32, + #[error("Invalid FFI u64 type, expected unsigned integer")] + InvalidU64, + #[error("Invalid FFI i64 type, expected integer")] + InvalidI64, + #[error("Invalid FFI usize type, expected unsigned integer")] + InvalidUsize, + #[error("Invalid FFI isize type, expected integer")] + InvalidIsize, + #[error("Invalid FFI f32 type, expected number")] + InvalidF32, + #[error("Invalid FFI f64 type, expected number")] + InvalidF64, + #[error("Invalid FFI pointer type, expected null, or External")] + InvalidPointerType, + #[error( + "Invalid FFI buffer type, expected null, ArrayBuffer, or ArrayBufferView" + )] + InvalidBufferType, + #[error("Invalid FFI ArrayBufferView, expected data in the buffer")] + InvalidArrayBufferView, + #[error("Invalid FFI ArrayBuffer, expected data in buffer")] + InvalidArrayBuffer, + #[error("Invalid FFI struct type, expected ArrayBuffer, or ArrayBufferView")] + InvalidStructType, + #[error("Invalid FFI function type, expected null, or External")] + InvalidFunctionType, +} + pub struct OutBuffer(pub *mut u8); // SAFETY: OutBuffer is allocated by us in 00_ffi.js and is guaranteed to be @@ -126,9 +168,9 @@ unsafe impl Send for NativeValue {} #[inline] pub fn ffi_parse_bool_arg( arg: v8::Local, -) -> Result { +) -> Result { let bool_value = v8::Local::::try_from(arg) - .map_err(|_| type_error("Invalid FFI u8 type, expected boolean"))? + .map_err(|_| IRError::InvalidU8ExpectedBoolean)? .is_true(); Ok(NativeValue { bool_value }) } @@ -136,9 +178,9 @@ pub fn ffi_parse_bool_arg( #[inline] pub fn ffi_parse_u8_arg( arg: v8::Local, -) -> Result { +) -> Result { let u8_value = v8::Local::::try_from(arg) - .map_err(|_| type_error("Invalid FFI u8 type, expected unsigned integer"))? + .map_err(|_| IRError::InvalidU8ExpectedUnsignedInteger)? .value() as u8; Ok(NativeValue { u8_value }) } @@ -146,9 +188,9 @@ pub fn ffi_parse_u8_arg( #[inline] pub fn ffi_parse_i8_arg( arg: v8::Local, -) -> Result { +) -> Result { let i8_value = v8::Local::::try_from(arg) - .map_err(|_| type_error("Invalid FFI i8 type, expected integer"))? + .map_err(|_| IRError::InvalidI8)? .value() as i8; Ok(NativeValue { i8_value }) } @@ -156,9 +198,9 @@ pub fn ffi_parse_i8_arg( #[inline] pub fn ffi_parse_u16_arg( arg: v8::Local, -) -> Result { +) -> Result { let u16_value = v8::Local::::try_from(arg) - .map_err(|_| type_error("Invalid FFI u16 type, expected unsigned integer"))? + .map_err(|_| IRError::InvalidU16)? .value() as u16; Ok(NativeValue { u16_value }) } @@ -166,9 +208,9 @@ pub fn ffi_parse_u16_arg( #[inline] pub fn ffi_parse_i16_arg( arg: v8::Local, -) -> Result { +) -> Result { let i16_value = v8::Local::::try_from(arg) - .map_err(|_| type_error("Invalid FFI i16 type, expected integer"))? + .map_err(|_| IRError::InvalidI16)? .value() as i16; Ok(NativeValue { i16_value }) } @@ -176,9 +218,9 @@ pub fn ffi_parse_i16_arg( #[inline] pub fn ffi_parse_u32_arg( arg: v8::Local, -) -> Result { +) -> Result { let u32_value = v8::Local::::try_from(arg) - .map_err(|_| type_error("Invalid FFI u32 type, expected unsigned integer"))? + .map_err(|_| IRError::InvalidU32)? .value(); Ok(NativeValue { u32_value }) } @@ -186,9 +228,9 @@ pub fn ffi_parse_u32_arg( #[inline] pub fn ffi_parse_i32_arg( arg: v8::Local, -) -> Result { +) -> Result { let i32_value = v8::Local::::try_from(arg) - .map_err(|_| type_error("Invalid FFI i32 type, expected integer"))? + .map_err(|_| IRError::InvalidI32)? .value(); Ok(NativeValue { i32_value }) } @@ -197,7 +239,7 @@ pub fn ffi_parse_i32_arg( pub fn ffi_parse_u64_arg( scope: &mut v8::HandleScope, arg: v8::Local, -) -> Result { +) -> Result { // Order of checking: // 1. BigInt: Uncommon and not supported by Fast API, so optimise slow call for this case. // 2. Number: Common, supported by Fast API, so let that be the optimal case. @@ -207,9 +249,7 @@ pub fn ffi_parse_u64_arg( } else if let Ok(value) = v8::Local::::try_from(arg) { value.integer_value(scope).unwrap() as u64 } else { - return Err(type_error( - "Invalid FFI u64 type, expected unsigned integer", - )); + return Err(IRError::InvalidU64); }; Ok(NativeValue { u64_value }) } @@ -218,7 +258,7 @@ pub fn ffi_parse_u64_arg( pub fn ffi_parse_i64_arg( scope: &mut v8::HandleScope, arg: v8::Local, -) -> Result { +) -> Result { // Order of checking: // 1. BigInt: Uncommon and not supported by Fast API, so optimise slow call for this case. // 2. Number: Common, supported by Fast API, so let that be the optimal case. @@ -228,7 +268,7 @@ pub fn ffi_parse_i64_arg( } else if let Ok(value) = v8::Local::::try_from(arg) { value.integer_value(scope).unwrap() } else { - return Err(type_error("Invalid FFI i64 type, expected integer")); + return Err(IRError::InvalidI64); }; Ok(NativeValue { i64_value }) } @@ -237,7 +277,7 @@ pub fn ffi_parse_i64_arg( pub fn ffi_parse_usize_arg( scope: &mut v8::HandleScope, arg: v8::Local, -) -> Result { +) -> Result { // Order of checking: // 1. BigInt: Uncommon and not supported by Fast API, so optimise slow call for this case. // 2. Number: Common, supported by Fast API, so let that be the optimal case. @@ -247,7 +287,7 @@ pub fn ffi_parse_usize_arg( } else if let Ok(value) = v8::Local::::try_from(arg) { value.integer_value(scope).unwrap() as usize } else { - return Err(type_error("Invalid FFI usize type, expected integer")); + return Err(IRError::InvalidUsize); }; Ok(NativeValue { usize_value }) } @@ -256,7 +296,7 @@ pub fn ffi_parse_usize_arg( pub fn ffi_parse_isize_arg( scope: &mut v8::HandleScope, arg: v8::Local, -) -> Result { +) -> Result { // Order of checking: // 1. BigInt: Uncommon and not supported by Fast API, so optimise slow call for this case. // 2. Number: Common, supported by Fast API, so let that be the optimal case. @@ -266,7 +306,7 @@ pub fn ffi_parse_isize_arg( } else if let Ok(value) = v8::Local::::try_from(arg) { value.integer_value(scope).unwrap() as isize } else { - return Err(type_error("Invalid FFI isize type, expected integer")); + return Err(IRError::InvalidIsize); }; Ok(NativeValue { isize_value }) } @@ -274,9 +314,9 @@ pub fn ffi_parse_isize_arg( #[inline] pub fn ffi_parse_f32_arg( arg: v8::Local, -) -> Result { +) -> Result { let f32_value = v8::Local::::try_from(arg) - .map_err(|_| type_error("Invalid FFI f32 type, expected number"))? + .map_err(|_| IRError::InvalidF32)? .value() as f32; Ok(NativeValue { f32_value }) } @@ -284,9 +324,9 @@ pub fn ffi_parse_f32_arg( #[inline] pub fn ffi_parse_f64_arg( arg: v8::Local, -) -> Result { +) -> Result { let f64_value = v8::Local::::try_from(arg) - .map_err(|_| type_error("Invalid FFI f64 type, expected number"))? + .map_err(|_| IRError::InvalidF64)? .value(); Ok(NativeValue { f64_value }) } @@ -295,15 +335,13 @@ pub fn ffi_parse_f64_arg( pub fn ffi_parse_pointer_arg( _scope: &mut v8::HandleScope, arg: v8::Local, -) -> Result { +) -> Result { let pointer = if let Ok(value) = v8::Local::::try_from(arg) { value.value() } else if arg.is_null() { ptr::null_mut() } else { - return Err(type_error( - "Invalid FFI pointer type, expected null, or External", - )); + return Err(IRError::InvalidPointerType); }; Ok(NativeValue { pointer }) } @@ -312,7 +350,7 @@ pub fn ffi_parse_pointer_arg( pub fn ffi_parse_buffer_arg( scope: &mut v8::HandleScope, arg: v8::Local, -) -> Result { +) -> Result { // Order of checking: // 1. ArrayBuffer: Fairly common and not supported by Fast API, optimise this case. // 2. ArrayBufferView: Common and supported by Fast API @@ -328,9 +366,7 @@ pub fn ffi_parse_buffer_arg( let byte_offset = value.byte_offset(); let pointer = value .buffer(scope) - .ok_or_else(|| { - type_error("Invalid FFI ArrayBufferView, expected data in the buffer") - })? + .ok_or(IRError::InvalidArrayBufferView)? .data(); if let Some(non_null) = pointer { // SAFETY: Pointer is non-null, and V8 guarantees that the byte_offset @@ -342,9 +378,7 @@ pub fn ffi_parse_buffer_arg( } else if arg.is_null() { ptr::null_mut() } else { - return Err(type_error( - "Invalid FFI buffer type, expected null, ArrayBuffer, or ArrayBufferView", - )); + return Err(IRError::InvalidBufferType); }; Ok(NativeValue { pointer }) } @@ -353,7 +387,7 @@ pub fn ffi_parse_buffer_arg( pub fn ffi_parse_struct_arg( scope: &mut v8::HandleScope, arg: v8::Local, -) -> Result { +) -> Result { // Order of checking: // 1. ArrayBuffer: Fairly common and not supported by Fast API, optimise this case. // 2. ArrayBufferView: Common and supported by Fast API @@ -362,31 +396,23 @@ pub fn ffi_parse_struct_arg( if let Some(non_null) = value.data() { non_null.as_ptr() } else { - return Err(type_error( - "Invalid FFI ArrayBuffer, expected data in buffer", - )); + return Err(IRError::InvalidArrayBuffer); } } else if let Ok(value) = v8::Local::::try_from(arg) { let byte_offset = value.byte_offset(); let pointer = value .buffer(scope) - .ok_or_else(|| { - type_error("Invalid FFI ArrayBufferView, expected data in the buffer") - })? + .ok_or(IRError::InvalidArrayBufferView)? .data(); if let Some(non_null) = pointer { // SAFETY: Pointer is non-null, and V8 guarantees that the byte_offset // is within the buffer backing store. unsafe { non_null.as_ptr().add(byte_offset) } } else { - return Err(type_error( - "Invalid FFI ArrayBufferView, expected data in buffer", - )); + return Err(IRError::InvalidArrayBufferView); } } else { - return Err(type_error( - "Invalid FFI struct type, expected ArrayBuffer, or ArrayBufferView", - )); + return Err(IRError::InvalidStructType); }; Ok(NativeValue { pointer }) } @@ -395,15 +421,13 @@ pub fn ffi_parse_struct_arg( pub fn ffi_parse_function_arg( _scope: &mut v8::HandleScope, arg: v8::Local, -) -> Result { +) -> Result { let pointer = if let Ok(value) = v8::Local::::try_from(arg) { value.value() } else if arg.is_null() { ptr::null_mut() } else { - return Err(type_error( - "Invalid FFI function type, expected null, or External", - )); + return Err(IRError::InvalidFunctionType); }; Ok(NativeValue { pointer }) } @@ -412,7 +436,7 @@ pub fn ffi_parse_args<'scope>( scope: &mut v8::HandleScope<'scope>, args: v8::Local, parameter_types: &[NativeType], -) -> Result, AnyError> +) -> Result, IRError> where 'scope: 'scope, { diff --git a/ext/ffi/lib.rs b/ext/ffi/lib.rs index 77ec3c85e3b131..73ec7757abc4e9 100644 --- a/ext/ffi/lib.rs +++ b/ext/ffi/lib.rs @@ -1,7 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::AnyError; - use std::mem::size_of; use std::os::raw::c_char; use std::os::raw::c_short; @@ -29,6 +27,14 @@ use repr::*; use symbol::NativeType; use symbol::Symbol; +pub use call::CallError; +pub use callback::CallbackError; +use deno_permissions::PermissionCheckError; +pub use dlfcn::DlfcnError; +pub use ir::IRError; +pub use r#static::StaticError; +pub use repr::ReprError; + #[cfg(not(target_pointer_width = "64"))] compile_error!("platform not supported"); @@ -41,17 +47,17 @@ const _: () = { pub const UNSTABLE_FEATURE_NAME: &str = "ffi"; pub trait FfiPermissions { - fn check_partial_no_path(&mut self) -> Result<(), AnyError>; + fn check_partial_no_path(&mut self) -> Result<(), PermissionCheckError>; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] fn check_partial_with_path( &mut self, path: &str, - ) -> Result; + ) -> Result; } impl FfiPermissions for deno_permissions::PermissionsContainer { #[inline(always)] - fn check_partial_no_path(&mut self) -> Result<(), AnyError> { + fn check_partial_no_path(&mut self) -> Result<(), PermissionCheckError> { deno_permissions::PermissionsContainer::check_ffi_partial_no_path(self) } @@ -59,7 +65,7 @@ impl FfiPermissions for deno_permissions::PermissionsContainer { fn check_partial_with_path( &mut self, path: &str, - ) -> Result { + ) -> Result { deno_permissions::PermissionsContainer::check_ffi_partial_with_path( self, path, ) diff --git a/ext/ffi/repr.rs b/ext/ffi/repr.rs index 315e6d53bc4b17..eea15f3e9771dd 100644 --- a/ext/ffi/repr.rs +++ b/ext/ffi/repr.rs @@ -1,9 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use crate::FfiPermissions; -use deno_core::error::range_error; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::v8; use deno_core::OpState; @@ -12,11 +9,51 @@ use std::ffi::c_void; use std::ffi::CStr; use std::ptr; -#[op2(fast)] +#[derive(Debug, thiserror::Error)] +pub enum ReprError { + #[error("Invalid pointer to offset, pointer is null")] + InvalidOffset, + #[error("Invalid ArrayBuffer pointer, pointer is null")] + InvalidArrayBuffer, + #[error("Destination length is smaller than source length")] + DestinationLengthTooShort, + #[error("Invalid CString pointer, pointer is null")] + InvalidCString, + #[error("Invalid CString pointer, string exceeds max length")] + CStringTooLong, + #[error("Invalid bool pointer, pointer is null")] + InvalidBool, + #[error("Invalid u8 pointer, pointer is null")] + InvalidU8, + #[error("Invalid i8 pointer, pointer is null")] + InvalidI8, + #[error("Invalid u16 pointer, pointer is null")] + InvalidU16, + #[error("Invalid i16 pointer, pointer is null")] + InvalidI16, + #[error("Invalid u32 pointer, pointer is null")] + InvalidU32, + #[error("Invalid i32 pointer, pointer is null")] + InvalidI32, + #[error("Invalid u64 pointer, pointer is null")] + InvalidU64, + #[error("Invalid i64 pointer, pointer is null")] + InvalidI64, + #[error("Invalid f32 pointer, pointer is null")] + InvalidF32, + #[error("Invalid f64 pointer, pointer is null")] + InvalidF64, + #[error("Invalid pointer pointer, pointer is null")] + InvalidPointer, + #[error(transparent)] + Permission(#[from] deno_permissions::PermissionCheckError), +} + +#[op2(fast, stack_trace)] pub fn op_ffi_ptr_create( state: &mut OpState, #[bigint] ptr_number: usize, -) -> Result<*mut c_void, AnyError> +) -> Result<*mut c_void, ReprError> where FP: FfiPermissions + 'static, { @@ -26,12 +63,12 @@ where Ok(ptr_number as *mut c_void) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_ffi_ptr_equals( state: &mut OpState, a: *const c_void, b: *const c_void, -) -> Result +) -> Result where FP: FfiPermissions + 'static, { @@ -41,11 +78,11 @@ where Ok(a == b) } -#[op2] +#[op2(stack_trace)] pub fn op_ffi_ptr_of( state: &mut OpState, #[anybuffer] buf: *const u8, -) -> Result<*mut c_void, AnyError> +) -> Result<*mut c_void, ReprError> where FP: FfiPermissions + 'static, { @@ -55,11 +92,11 @@ where Ok(buf as *mut c_void) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_ffi_ptr_of_exact( state: &mut OpState, buf: v8::Local, -) -> Result<*mut c_void, AnyError> +) -> Result<*mut c_void, ReprError> where FP: FfiPermissions + 'static, { @@ -75,12 +112,12 @@ where Ok(buf.as_ptr() as _) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_ffi_ptr_offset( state: &mut OpState, ptr: *mut c_void, #[number] offset: isize, -) -> Result<*mut c_void, AnyError> +) -> Result<*mut c_void, ReprError> where FP: FfiPermissions + 'static, { @@ -88,7 +125,7 @@ where permissions.check_partial_no_path()?; if ptr.is_null() { - return Err(type_error("Invalid pointer to offset, pointer is null")); + return Err(ReprError::InvalidOffset); } // TODO(mmastrac): Create a RawPointer that can safely do pointer math. @@ -105,12 +142,12 @@ unsafe extern "C" fn noop_deleter_callback( ) { } -#[op2(fast)] +#[op2(fast, stack_trace)] #[bigint] pub fn op_ffi_ptr_value( state: &mut OpState, ptr: *mut c_void, -) -> Result +) -> Result where FP: FfiPermissions + 'static, { @@ -120,14 +157,14 @@ where Ok(ptr as usize) } -#[op2] +#[op2(stack_trace)] pub fn op_ffi_get_buf( scope: &mut v8::HandleScope<'scope>, state: &mut OpState, ptr: *mut c_void, #[number] offset: isize, #[number] len: usize, -) -> Result, AnyError> +) -> Result, ReprError> where FP: FfiPermissions + 'static, { @@ -135,7 +172,7 @@ where permissions.check_partial_no_path()?; if ptr.is_null() { - return Err(type_error("Invalid ArrayBuffer pointer, pointer is null")); + return Err(ReprError::InvalidArrayBuffer); } // SAFETY: Trust the user to have provided a real pointer, offset, and a valid matching size to it. Since this is a foreign pointer, we should not do any deletion. @@ -144,7 +181,7 @@ where ptr.offset(offset), len, noop_deleter_callback, - std::ptr::null_mut(), + ptr::null_mut(), ) } .make_shared(); @@ -152,14 +189,14 @@ where Ok(array_buffer) } -#[op2] +#[op2(stack_trace)] pub fn op_ffi_buf_copy_into( state: &mut OpState, src: *mut c_void, #[number] offset: isize, #[anybuffer] dst: &mut [u8], #[number] len: usize, -) -> Result<(), AnyError> +) -> Result<(), ReprError> where FP: FfiPermissions + 'static, { @@ -167,11 +204,9 @@ where permissions.check_partial_no_path()?; if src.is_null() { - Err(type_error("Invalid ArrayBuffer pointer, pointer is null")) + Err(ReprError::InvalidArrayBuffer) } else if dst.len() < len { - Err(range_error( - "Destination length is smaller than source length", - )) + Err(ReprError::DestinationLengthTooShort) } else { let src = src as *const c_void; @@ -184,13 +219,13 @@ where } } -#[op2] +#[op2(stack_trace)] pub fn op_ffi_cstr_read( scope: &mut v8::HandleScope<'scope>, state: &mut OpState, ptr: *mut c_void, #[number] offset: isize, -) -> Result, AnyError> +) -> Result, ReprError> where FP: FfiPermissions + 'static, { @@ -198,25 +233,23 @@ where permissions.check_partial_no_path()?; if ptr.is_null() { - return Err(type_error("Invalid CString pointer, pointer is null")); + return Err(ReprError::InvalidCString); } let cstr = // SAFETY: Pointer and offset are user provided. unsafe { CStr::from_ptr(ptr.offset(offset) as *const c_char) }.to_bytes(); let value = v8::String::new_from_utf8(scope, cstr, v8::NewStringType::Normal) - .ok_or_else(|| { - type_error("Invalid CString pointer, string exceeds max length") - })?; + .ok_or_else(|| ReprError::CStringTooLong)?; Ok(value) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_ffi_read_bool( state: &mut OpState, ptr: *mut c_void, #[number] offset: isize, -) -> Result +) -> Result where FP: FfiPermissions + 'static, { @@ -224,19 +257,19 @@ where permissions.check_partial_no_path()?; if ptr.is_null() { - return Err(type_error("Invalid bool pointer, pointer is null")); + return Err(ReprError::InvalidBool); } // SAFETY: ptr and offset are user provided. Ok(unsafe { ptr::read_unaligned::(ptr.offset(offset) as *const bool) }) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_ffi_read_u8( state: &mut OpState, ptr: *mut c_void, #[number] offset: isize, -) -> Result +) -> Result where FP: FfiPermissions + 'static, { @@ -244,7 +277,7 @@ where permissions.check_partial_no_path()?; if ptr.is_null() { - return Err(type_error("Invalid u8 pointer, pointer is null")); + return Err(ReprError::InvalidU8); } // SAFETY: ptr and offset are user provided. @@ -253,12 +286,12 @@ where }) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_ffi_read_i8( state: &mut OpState, ptr: *mut c_void, #[number] offset: isize, -) -> Result +) -> Result where FP: FfiPermissions + 'static, { @@ -266,7 +299,7 @@ where permissions.check_partial_no_path()?; if ptr.is_null() { - return Err(type_error("Invalid i8 pointer, pointer is null")); + return Err(ReprError::InvalidI8); } // SAFETY: ptr and offset are user provided. @@ -275,12 +308,12 @@ where }) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_ffi_read_u16( state: &mut OpState, ptr: *mut c_void, #[number] offset: isize, -) -> Result +) -> Result where FP: FfiPermissions + 'static, { @@ -288,7 +321,7 @@ where permissions.check_partial_no_path()?; if ptr.is_null() { - return Err(type_error("Invalid u16 pointer, pointer is null")); + return Err(ReprError::InvalidU16); } // SAFETY: ptr and offset are user provided. @@ -297,12 +330,12 @@ where }) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_ffi_read_i16( state: &mut OpState, ptr: *mut c_void, #[number] offset: isize, -) -> Result +) -> Result where FP: FfiPermissions + 'static, { @@ -310,7 +343,7 @@ where permissions.check_partial_no_path()?; if ptr.is_null() { - return Err(type_error("Invalid i16 pointer, pointer is null")); + return Err(ReprError::InvalidI16); } // SAFETY: ptr and offset are user provided. @@ -319,12 +352,12 @@ where }) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_ffi_read_u32( state: &mut OpState, ptr: *mut c_void, #[number] offset: isize, -) -> Result +) -> Result where FP: FfiPermissions + 'static, { @@ -332,19 +365,19 @@ where permissions.check_partial_no_path()?; if ptr.is_null() { - return Err(type_error("Invalid u32 pointer, pointer is null")); + return Err(ReprError::InvalidU32); } // SAFETY: ptr and offset are user provided. Ok(unsafe { ptr::read_unaligned::(ptr.offset(offset) as *const u32) }) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_ffi_read_i32( state: &mut OpState, ptr: *mut c_void, #[number] offset: isize, -) -> Result +) -> Result where FP: FfiPermissions + 'static, { @@ -352,14 +385,14 @@ where permissions.check_partial_no_path()?; if ptr.is_null() { - return Err(type_error("Invalid i32 pointer, pointer is null")); + return Err(ReprError::InvalidI32); } // SAFETY: ptr and offset are user provided. Ok(unsafe { ptr::read_unaligned::(ptr.offset(offset) as *const i32) }) } -#[op2(fast)] +#[op2(fast, stack_trace)] #[bigint] pub fn op_ffi_read_u64( state: &mut OpState, @@ -367,7 +400,7 @@ pub fn op_ffi_read_u64( // Note: The representation of 64-bit integers is function-wide. We cannot // choose to take this parameter as a number while returning a bigint. #[bigint] offset: isize, -) -> Result +) -> Result where FP: FfiPermissions + 'static, { @@ -375,7 +408,7 @@ where permissions.check_partial_no_path()?; if ptr.is_null() { - return Err(type_error("Invalid u64 pointer, pointer is null")); + return Err(ReprError::InvalidU64); } let value = @@ -385,7 +418,7 @@ where Ok(value) } -#[op2(fast)] +#[op2(fast, stack_trace)] #[bigint] pub fn op_ffi_read_i64( state: &mut OpState, @@ -393,7 +426,7 @@ pub fn op_ffi_read_i64( // Note: The representation of 64-bit integers is function-wide. We cannot // choose to take this parameter as a number while returning a bigint. #[bigint] offset: isize, -) -> Result +) -> Result where FP: FfiPermissions + 'static, { @@ -401,7 +434,7 @@ where permissions.check_partial_no_path()?; if ptr.is_null() { - return Err(type_error("Invalid i64 pointer, pointer is null")); + return Err(ReprError::InvalidI64); } let value = @@ -411,12 +444,12 @@ where Ok(value) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_ffi_read_f32( state: &mut OpState, ptr: *mut c_void, #[number] offset: isize, -) -> Result +) -> Result where FP: FfiPermissions + 'static, { @@ -424,19 +457,19 @@ where permissions.check_partial_no_path()?; if ptr.is_null() { - return Err(type_error("Invalid f32 pointer, pointer is null")); + return Err(ReprError::InvalidF32); } // SAFETY: ptr and offset are user provided. Ok(unsafe { ptr::read_unaligned::(ptr.offset(offset) as *const f32) }) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_ffi_read_f64( state: &mut OpState, ptr: *mut c_void, #[number] offset: isize, -) -> Result +) -> Result where FP: FfiPermissions + 'static, { @@ -444,19 +477,19 @@ where permissions.check_partial_no_path()?; if ptr.is_null() { - return Err(type_error("Invalid f64 pointer, pointer is null")); + return Err(ReprError::InvalidF64); } // SAFETY: ptr and offset are user provided. Ok(unsafe { ptr::read_unaligned::(ptr.offset(offset) as *const f64) }) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_ffi_read_ptr( state: &mut OpState, ptr: *mut c_void, #[number] offset: isize, -) -> Result<*mut c_void, AnyError> +) -> Result<*mut c_void, ReprError> where FP: FfiPermissions + 'static, { @@ -464,7 +497,7 @@ where permissions.check_partial_no_path()?; if ptr.is_null() { - return Err(type_error("Invalid pointer pointer, pointer is null")); + return Err(ReprError::InvalidPointer); } // SAFETY: ptr and offset are user provided. diff --git a/ext/ffi/static.rs b/ext/ffi/static.rs index f08605754b2c27..61b40593367f31 100644 --- a/ext/ffi/static.rs +++ b/ext/ffi/static.rs @@ -2,14 +2,24 @@ use crate::dlfcn::DynamicLibraryResource; use crate::symbol::NativeType; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::v8; use deno_core::OpState; use deno_core::ResourceId; use std::ptr; +#[derive(Debug, thiserror::Error)] +pub enum StaticError { + #[error(transparent)] + Dlfcn(super::DlfcnError), + #[error("Invalid FFI static type 'void'")] + InvalidTypeVoid, + #[error("Invalid FFI static type 'struct'")] + InvalidTypeStruct, + #[error(transparent)] + Resource(deno_core::error::AnyError), +} + #[op2] pub fn op_ffi_get_static<'scope>( scope: &mut v8::HandleScope<'scope>, @@ -18,24 +28,27 @@ pub fn op_ffi_get_static<'scope>( #[string] name: String, #[serde] static_type: NativeType, optional: bool, -) -> Result, AnyError> { - let resource = state.resource_table.get::(rid)?; +) -> Result, StaticError> { + let resource = state + .resource_table + .get::(rid) + .map_err(StaticError::Resource)?; let data_ptr = match resource.get_static(name) { - Ok(data_ptr) => Ok(data_ptr), + Ok(data_ptr) => data_ptr, Err(err) => { if optional { let null: v8::Local = v8::null(scope).into(); return Ok(null); } else { - Err(err) + return Err(StaticError::Dlfcn(err)); } } - }?; + }; Ok(match static_type { NativeType::Void => { - return Err(type_error("Invalid FFI static type 'void'")); + return Err(StaticError::InvalidTypeVoid); } NativeType::Bool => { // SAFETY: ptr is user provided @@ -132,7 +145,7 @@ pub fn op_ffi_get_static<'scope>( external } NativeType::Struct(_) => { - return Err(type_error("Invalid FFI static type 'struct'")); + return Err(StaticError::InvalidTypeStruct); } }) } diff --git a/ext/fs/30_fs.js b/ext/fs/30_fs.js index c8e19ac7583130..40513e7e0239cb 100644 --- a/ext/fs/30_fs.js +++ b/ext/fs/30_fs.js @@ -346,9 +346,10 @@ const { 0: statStruct, 1: statBuf } = createByteStruct({ mtime: "date", atime: "date", birthtime: "date", + ctime: "date", dev: "u64", ino: "?u64", - mode: "?u64", + mode: "u64", nlink: "?u64", uid: "?u64", gid: "?u64", @@ -377,9 +378,10 @@ function parseFileInfo(response) { birthtime: response.birthtimeSet === true ? new Date(response.birthtime) : null, + ctime: response.ctimeSet === true ? new Date(response.ctime) : null, dev: response.dev, + mode: response.mode, ino: unix ? response.ino : null, - mode: unix ? response.mode : null, nlink: unix ? response.nlink : null, uid: unix ? response.uid : null, gid: unix ? response.gid : null, diff --git a/ext/fs/Cargo.toml b/ext/fs/Cargo.toml index 894b307efa9248..3fbcef44750f52 100644 --- a/ext/fs/Cargo.toml +++ b/ext/fs/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_fs" -version = "0.78.0" +version = "0.90.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -19,17 +19,20 @@ sync_fs = [] [dependencies] async-trait.workspace = true base32.workspace = true +boxed_error.workspace = true deno_core.workspace = true deno_io.workspace = true +deno_path_util.workspace = true deno_permissions.workspace = true filetime.workspace = true libc.workspace = true rand.workspace = true rayon = "1.8.0" serde.workspace = true +thiserror.workspace = true [target.'cfg(unix)'.dependencies] -nix.workspace = true +nix = { workspace = true, features = ["fs", "user"] } [target.'cfg(windows)'.dependencies] winapi = { workspace = true, features = ["winbase"] } diff --git a/ext/fs/in_memory_fs.rs b/ext/fs/in_memory_fs.rs index 027539e8492b45..b79b0ae984d47e 100644 --- a/ext/fs/in_memory_fs.rs +++ b/ext/fs/in_memory_fs.rs @@ -3,6 +3,7 @@ // Allow using Arc for this module. #![allow(clippy::disallowed_types)] +use std::borrow::Cow; use std::collections::hash_map::Entry; use std::collections::HashMap; use std::io::Error; @@ -12,12 +13,12 @@ use std::path::PathBuf; use std::rc::Rc; use std::sync::Arc; -use deno_core::normalize_path; use deno_core::parking_lot::Mutex; use deno_io::fs::File; use deno_io::fs::FsError; use deno_io::fs::FsResult; use deno_io::fs::FsStat; +use deno_path_util::normalize_path; use crate::interface::AccessCheckCb; use crate::interface::FsDirEntry; @@ -44,7 +45,7 @@ impl InMemoryFs { pub fn setup_text_files(&self, files: Vec<(String, String)>) { for (path, text) in files { let path = PathBuf::from(path); - self.mkdir_sync(path.parent().unwrap(), true, 0).unwrap(); + self.mkdir_sync(path.parent().unwrap(), true, None).unwrap(); self .write_file_sync( &path, @@ -101,7 +102,7 @@ impl FileSystem for InMemoryFs { &self, path: &Path, recursive: bool, - _mode: u32, + _mode: Option, ) -> FsResult<()> { let path = normalize_path(path); @@ -119,7 +120,7 @@ impl FileSystem for InMemoryFs { }, None => { if recursive { - self.mkdir_sync(parent, true, 0)?; + self.mkdir_sync(parent, true, None)?; } else { return Err(FsError::Io(Error::new( ErrorKind::NotFound, @@ -149,7 +150,7 @@ impl FileSystem for InMemoryFs { &self, path: PathBuf, recursive: bool, - mode: u32, + mode: Option, ) -> FsResult<()> { self.mkdir_sync(&path, recursive, mode) } @@ -229,6 +230,7 @@ impl FileSystem for InMemoryFs { mtime: None, atime: None, birthtime: None, + ctime: None, dev: 0, ino: 0, mode: 0, @@ -251,6 +253,7 @@ impl FileSystem for InMemoryFs { mtime: None, atime: None, birthtime: None, + ctime: None, dev: 0, ino: 0, mode: 0, @@ -455,11 +458,11 @@ impl FileSystem for InMemoryFs { &self, path: &Path, _access_check: Option, - ) -> FsResult> { + ) -> FsResult> { let entry = self.get_entry(path); match entry { Some(entry) => match &*entry { - PathEntry::File(data) => Ok(data.clone()), + PathEntry::File(data) => Ok(Cow::Owned(data.clone())), PathEntry::Dir => Err(FsError::Io(Error::new( ErrorKind::InvalidInput, "Is a directory", @@ -472,7 +475,7 @@ impl FileSystem for InMemoryFs { &'a self, path: PathBuf, access_check: Option>, - ) -> FsResult> { + ) -> FsResult> { self.read_file_sync(&path, access_check) } } diff --git a/ext/fs/interface.rs b/ext/fs/interface.rs index af4beb248bfb2b..28a49c5d9b887a 100644 --- a/ext/fs/interface.rs +++ b/ext/fs/interface.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use core::str; use std::borrow::Cow; use std::path::Path; use std::path::PathBuf; @@ -121,13 +122,17 @@ pub trait FileSystem: std::fmt::Debug + MaybeSend + MaybeSync { access_check: Option>, ) -> FsResult>; - fn mkdir_sync(&self, path: &Path, recursive: bool, mode: u32) - -> FsResult<()>; + fn mkdir_sync( + &self, + path: &Path, + recursive: bool, + mode: Option, + ) -> FsResult<()>; async fn mkdir_async( &self, path: PathBuf, recursive: bool, - mode: u32, + mode: Option, ) -> FsResult<()>; fn chmod_sync(&self, path: &Path, mode: u32) -> FsResult<()>; @@ -284,7 +289,7 @@ pub trait FileSystem: std::fmt::Debug + MaybeSend + MaybeSync { &self, path: &Path, access_check: Option, - ) -> FsResult> { + ) -> FsResult> { let options = OpenOptions::read(); let file = self.open_sync(path, options, access_check)?; let buf = file.read_all_sync()?; @@ -294,7 +299,7 @@ pub trait FileSystem: std::fmt::Debug + MaybeSend + MaybeSync { &'a self, path: PathBuf, access_check: Option>, - ) -> FsResult> { + ) -> FsResult> { let options = OpenOptions::read(); let file = self.open_async(path, options, access_check).await?; let buf = file.read_all_async().await?; @@ -323,17 +328,25 @@ pub trait FileSystem: std::fmt::Debug + MaybeSend + MaybeSync { &self, path: &Path, access_check: Option, - ) -> FsResult { + ) -> FsResult> { let buf = self.read_file_sync(path, access_check)?; - Ok(string_from_utf8_lossy(buf)) + Ok(string_from_cow_utf8_lossy(buf)) } async fn read_text_file_lossy_async<'a>( &'a self, path: PathBuf, access_check: Option>, - ) -> FsResult { + ) -> FsResult> { let buf = self.read_file_async(path, access_check).await?; - Ok(string_from_utf8_lossy(buf)) + Ok(string_from_cow_utf8_lossy(buf)) + } +} + +#[inline(always)] +fn string_from_cow_utf8_lossy(buf: Cow<'static, [u8]>) -> Cow<'static, str> { + match buf { + Cow::Owned(buf) => Cow::Owned(string_from_utf8_lossy(buf)), + Cow::Borrowed(buf) => String::from_utf8_lossy(buf), } } diff --git a/ext/fs/lib.rs b/ext/fs/lib.rs index bd49078b2e94db..26fac1e79f5b02 100644 --- a/ext/fs/lib.rs +++ b/ext/fs/lib.rs @@ -14,14 +14,18 @@ pub use crate::interface::FileSystemRc; pub use crate::interface::FsDirEntry; pub use crate::interface::FsFileType; pub use crate::interface::OpenOptions; +pub use crate::ops::FsOpsError; +pub use crate::ops::FsOpsErrorKind; +pub use crate::ops::OperationError; +pub use crate::ops::V8MaybeStaticStr; pub use crate::std_fs::RealFs; pub use crate::sync::MaybeSend; pub use crate::sync::MaybeSync; use crate::ops::*; -use deno_core::error::AnyError; use deno_io::fs::FsError; +use deno_permissions::PermissionCheckError; use std::borrow::Cow; use std::path::Path; use std::path::PathBuf; @@ -40,45 +44,51 @@ pub trait FsPermissions { &mut self, path: &str, api_name: &str, - ) -> Result; + ) -> Result; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] fn check_read_path<'a>( &mut self, path: &'a Path, api_name: &str, - ) -> Result, AnyError>; - fn check_read_all(&mut self, api_name: &str) -> Result<(), AnyError>; + ) -> Result, PermissionCheckError>; + fn check_read_all( + &mut self, + api_name: &str, + ) -> Result<(), PermissionCheckError>; fn check_read_blind( &mut self, p: &Path, display: &str, api_name: &str, - ) -> Result<(), AnyError>; + ) -> Result<(), PermissionCheckError>; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] fn check_write( &mut self, path: &str, api_name: &str, - ) -> Result; + ) -> Result; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] fn check_write_path<'a>( &mut self, path: &'a Path, api_name: &str, - ) -> Result, AnyError>; + ) -> Result, PermissionCheckError>; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] fn check_write_partial( &mut self, path: &str, api_name: &str, - ) -> Result; - fn check_write_all(&mut self, api_name: &str) -> Result<(), AnyError>; + ) -> Result; + fn check_write_all( + &mut self, + api_name: &str, + ) -> Result<(), PermissionCheckError>; fn check_write_blind( &mut self, p: &Path, display: &str, api_name: &str, - ) -> Result<(), AnyError>; + ) -> Result<(), PermissionCheckError>; fn check<'a>( &mut self, @@ -138,7 +148,7 @@ impl FsPermissions for deno_permissions::PermissionsContainer { &mut self, path: &str, api_name: &str, - ) -> Result { + ) -> Result { deno_permissions::PermissionsContainer::check_read(self, path, api_name) } @@ -146,7 +156,7 @@ impl FsPermissions for deno_permissions::PermissionsContainer { &mut self, path: &'a Path, api_name: &str, - ) -> Result, AnyError> { + ) -> Result, PermissionCheckError> { deno_permissions::PermissionsContainer::check_read_path( self, path, @@ -158,7 +168,7 @@ impl FsPermissions for deno_permissions::PermissionsContainer { path: &Path, display: &str, api_name: &str, - ) -> Result<(), AnyError> { + ) -> Result<(), PermissionCheckError> { deno_permissions::PermissionsContainer::check_read_blind( self, path, display, api_name, ) @@ -168,7 +178,7 @@ impl FsPermissions for deno_permissions::PermissionsContainer { &mut self, path: &str, api_name: &str, - ) -> Result { + ) -> Result { deno_permissions::PermissionsContainer::check_write(self, path, api_name) } @@ -176,7 +186,7 @@ impl FsPermissions for deno_permissions::PermissionsContainer { &mut self, path: &'a Path, api_name: &str, - ) -> Result, AnyError> { + ) -> Result, PermissionCheckError> { deno_permissions::PermissionsContainer::check_write_path( self, path, api_name, ) @@ -186,7 +196,7 @@ impl FsPermissions for deno_permissions::PermissionsContainer { &mut self, path: &str, api_name: &str, - ) -> Result { + ) -> Result { deno_permissions::PermissionsContainer::check_write_partial( self, path, api_name, ) @@ -197,17 +207,23 @@ impl FsPermissions for deno_permissions::PermissionsContainer { p: &Path, display: &str, api_name: &str, - ) -> Result<(), AnyError> { + ) -> Result<(), PermissionCheckError> { deno_permissions::PermissionsContainer::check_write_blind( self, p, display, api_name, ) } - fn check_read_all(&mut self, api_name: &str) -> Result<(), AnyError> { + fn check_read_all( + &mut self, + api_name: &str, + ) -> Result<(), PermissionCheckError> { deno_permissions::PermissionsContainer::check_read_all(self, api_name) } - fn check_write_all(&mut self, api_name: &str) -> Result<(), AnyError> { + fn check_write_all( + &mut self, + api_name: &str, + ) -> Result<(), PermissionCheckError> { deno_permissions::PermissionsContainer::check_write_all(self, api_name) } } diff --git a/ext/fs/ops.rs b/ext/fs/ops.rs index 150d3b95552110..5e64585e0c051e 100644 --- a/ext/fs/ops.rs +++ b/ext/fs/ops.rs @@ -1,6 +1,9 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::borrow::Cow; use std::cell::RefCell; +use std::error::Error; +use std::fmt::Formatter; use std::io; use std::io::SeekFrom; use std::path::Path; @@ -8,31 +11,96 @@ use std::path::PathBuf; use std::path::StripPrefixError; use std::rc::Rc; -use deno_core::anyhow::bail; -use deno_core::error::custom_error; -use deno_core::error::type_error; -use deno_core::error::AnyError; +use crate::interface::AccessCheckFn; +use crate::interface::FileSystemRc; +use crate::interface::FsDirEntry; +use crate::interface::FsFileType; +use crate::FsPermissions; +use crate::OpenOptions; +use boxed_error::Boxed; use deno_core::op2; +use deno_core::v8; use deno_core::CancelFuture; use deno_core::CancelHandle; +use deno_core::FastString; use deno_core::JsBuffer; use deno_core::OpState; use deno_core::ResourceId; use deno_core::ToJsBuffer; +use deno_core::ToV8; use deno_io::fs::FileResource; use deno_io::fs::FsError; use deno_io::fs::FsStat; +use deno_permissions::PermissionCheckError; use rand::rngs::ThreadRng; use rand::thread_rng; use rand::Rng; use serde::Serialize; -use crate::interface::AccessCheckFn; -use crate::interface::FileSystemRc; -use crate::interface::FsDirEntry; -use crate::interface::FsFileType; -use crate::FsPermissions; -use crate::OpenOptions; +#[derive(Debug, Boxed)] +pub struct FsOpsError(pub Box); + +#[derive(Debug, thiserror::Error)] +pub enum FsOpsErrorKind { + #[error("{0}")] + Io(#[source] std::io::Error), + #[error("{0}")] + OperationError(#[source] OperationError), + #[error(transparent)] + Permission(#[from] PermissionCheckError), + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error("File name or path {0:?} is not valid UTF-8")] + InvalidUtf8(std::ffi::OsString), + #[error("{0}")] + StripPrefix(#[from] StripPrefixError), + #[error("{0}")] + Canceled(#[from] deno_core::Canceled), + #[error("Invalid seek mode: {0}")] + InvalidSeekMode(i32), + #[error("Invalid control character in prefix or suffix: {0:?}")] + InvalidControlCharacter(String), + #[error("Invalid character in prefix or suffix: {0:?}")] + InvalidCharacter(String), + #[cfg(windows)] + #[error("Invalid trailing character in suffix")] + InvalidTrailingCharacter, + #[error("Requires {err} access to {path}, {}", print_not_capable_info(*.standalone, .err))] + NotCapableAccess { + // NotCapable + standalone: bool, + err: &'static str, + path: String, + }, + #[error("permission denied: {0}")] + NotCapable(&'static str), // NotCapable + #[error(transparent)] + Other(deno_core::error::AnyError), +} + +impl From for FsOpsError { + fn from(err: FsError) -> Self { + match err { + FsError::Io(err) => FsOpsErrorKind::Io(err), + FsError::FileBusy => { + FsOpsErrorKind::Other(deno_core::error::resource_unavailable()) + } + FsError::NotSupported => { + FsOpsErrorKind::Other(deno_core::error::not_supported()) + } + FsError::NotCapable(err) => FsOpsErrorKind::NotCapable(err), + } + .into_box() + } +} + +fn print_not_capable_info(standalone: bool, err: &'static str) -> String { + if standalone { + format!("specify the required permissions during compilation using `deno compile --allow-{err}`") + } else { + format!("run again with the --allow-{err} flag") + } +} fn sync_permission_check<'a, P: FsPermissions + 'static>( permissions: &'a mut P, @@ -58,7 +126,7 @@ fn map_permission_error( operation: &'static str, error: FsError, path: &Path, -) -> AnyError { +) -> FsOpsError { match error { FsError::NotCapable(err) => { let path = format!("{path:?}"); @@ -67,14 +135,13 @@ fn map_permission_error( } else { (path.as_str(), "") }; - let msg = if deno_permissions::is_standalone() { - format!( - "Requires {err} access to {path}{truncated}, specify the required permissions during compilation using `deno compile --allow-{err}`") - } else { - format!( - "Requires {err} access to {path}{truncated}, run again with the --allow-{err} flag") - }; - custom_error("NotCapable", msg) + + FsOpsErrorKind::NotCapableAccess { + standalone: deno_permissions::is_standalone(), + err, + path: format!("{path}{truncated}"), + } + .into_box() } err => Err::<(), _>(err) .context_path(operation, path) @@ -83,9 +150,9 @@ fn map_permission_error( } } -#[op2] +#[op2(stack_trace)] #[string] -pub fn op_fs_cwd

(state: &mut OpState) -> Result +pub fn op_fs_cwd

(state: &mut OpState) -> Result where P: FsPermissions + 'static, { @@ -98,11 +165,11 @@ where Ok(path_str) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_fs_chdir

( state: &mut OpState, #[string] directory: &str, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -119,19 +186,19 @@ where pub fn op_fs_umask( state: &mut OpState, mask: Option, -) -> Result +) -> Result where { state.borrow::().umask(mask).context("umask") } -#[op2] +#[op2(stack_trace)] #[smi] pub fn op_fs_open_sync

( state: &mut OpState, #[string] path: String, #[serde] options: Option, -) -> Result +) -> Result where P: FsPermissions + 'static, { @@ -152,13 +219,13 @@ where Ok(rid) } -#[op2(async)] +#[op2(async, stack_trace)] #[smi] pub async fn op_fs_open_async

( state: Rc>, #[string] path: String, #[serde] options: Option, -) -> Result +) -> Result where P: FsPermissions + 'static, { @@ -180,13 +247,13 @@ where Ok(rid) } -#[op2] +#[op2(stack_trace)] pub fn op_fs_mkdir_sync

( state: &mut OpState, #[string] path: String, recursive: bool, mode: Option, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -197,19 +264,19 @@ where .check_write(&path, "Deno.mkdirSync()")?; let fs = state.borrow::(); - fs.mkdir_sync(&path, recursive, mode) + fs.mkdir_sync(&path, recursive, Some(mode)) .context_path("mkdir", &path)?; Ok(()) } -#[op2(async)] +#[op2(async, stack_trace)] pub async fn op_fs_mkdir_async

( state: Rc>, #[string] path: String, recursive: bool, mode: Option, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -221,19 +288,19 @@ where (state.borrow::().clone(), path) }; - fs.mkdir_async(path.clone(), recursive, mode) + fs.mkdir_async(path.clone(), recursive, Some(mode)) .await .context_path("mkdir", &path)?; Ok(()) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_fs_chmod_sync

( state: &mut OpState, #[string] path: String, mode: u32, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -245,12 +312,12 @@ where Ok(()) } -#[op2(async)] +#[op2(async, stack_trace)] pub async fn op_fs_chmod_async

( state: Rc>, #[string] path: String, mode: u32, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -265,13 +332,13 @@ where Ok(()) } -#[op2] +#[op2(stack_trace)] pub fn op_fs_chown_sync

( state: &mut OpState, #[string] path: String, uid: Option, gid: Option, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -284,13 +351,13 @@ where Ok(()) } -#[op2(async)] +#[op2(async, stack_trace)] pub async fn op_fs_chown_async

( state: Rc>, #[string] path: String, uid: Option, gid: Option, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -305,12 +372,12 @@ where Ok(()) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_fs_remove_sync

( state: &mut OpState, #[string] path: &str, recursive: bool, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -325,12 +392,12 @@ where Ok(()) } -#[op2(async)] +#[op2(async, stack_trace)] pub async fn op_fs_remove_async

( state: Rc>, #[string] path: String, recursive: bool, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -356,12 +423,12 @@ where Ok(()) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_fs_copy_file_sync

( state: &mut OpState, #[string] from: &str, #[string] to: &str, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -376,12 +443,12 @@ where Ok(()) } -#[op2(async)] +#[op2(async, stack_trace)] pub async fn op_fs_copy_file_async

( state: Rc>, #[string] from: String, #[string] to: String, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -400,12 +467,12 @@ where Ok(()) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_fs_stat_sync

( state: &mut OpState, #[string] path: String, #[buffer] stat_out_buf: &mut [u32], -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -419,12 +486,12 @@ where Ok(()) } -#[op2(async)] +#[op2(async, stack_trace)] #[serde] pub async fn op_fs_stat_async

( state: Rc>, #[string] path: String, -) -> Result +) -> Result where P: FsPermissions + 'static, { @@ -441,12 +508,12 @@ where Ok(SerializableStat::from(stat)) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_fs_lstat_sync

( state: &mut OpState, #[string] path: String, #[buffer] stat_out_buf: &mut [u32], -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -460,12 +527,12 @@ where Ok(()) } -#[op2(async)] +#[op2(async, stack_trace)] #[serde] pub async fn op_fs_lstat_async

( state: Rc>, #[string] path: String, -) -> Result +) -> Result where P: FsPermissions + 'static, { @@ -482,12 +549,12 @@ where Ok(SerializableStat::from(stat)) } -#[op2] +#[op2(stack_trace)] #[string] pub fn op_fs_realpath_sync

( state: &mut OpState, #[string] path: String, -) -> Result +) -> Result where P: FsPermissions + 'static, { @@ -505,12 +572,12 @@ where Ok(path_string) } -#[op2(async)] +#[op2(async, stack_trace)] #[string] pub async fn op_fs_realpath_async

( state: Rc>, #[string] path: String, -) -> Result +) -> Result where P: FsPermissions + 'static, { @@ -533,12 +600,12 @@ where Ok(path_string) } -#[op2] +#[op2(stack_trace)] #[serde] pub fn op_fs_read_dir_sync

( state: &mut OpState, #[string] path: String, -) -> Result, AnyError> +) -> Result, FsOpsError> where P: FsPermissions + 'static, { @@ -552,12 +619,12 @@ where Ok(entries) } -#[op2(async)] +#[op2(async, stack_trace)] #[serde] pub async fn op_fs_read_dir_async

( state: Rc>, #[string] path: String, -) -> Result, AnyError> +) -> Result, FsOpsError> where P: FsPermissions + 'static, { @@ -577,12 +644,12 @@ where Ok(entries) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_fs_rename_sync

( state: &mut OpState, #[string] oldpath: String, #[string] newpath: String, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -598,12 +665,12 @@ where Ok(()) } -#[op2(async)] +#[op2(async, stack_trace)] pub async fn op_fs_rename_async

( state: Rc>, #[string] oldpath: String, #[string] newpath: String, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -623,12 +690,12 @@ where Ok(()) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_fs_link_sync

( state: &mut OpState, #[string] oldpath: &str, #[string] newpath: &str, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -645,12 +712,12 @@ where Ok(()) } -#[op2(async)] +#[op2(async, stack_trace)] pub async fn op_fs_link_async

( state: Rc>, #[string] oldpath: String, #[string] newpath: String, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -671,13 +738,13 @@ where Ok(()) } -#[op2] +#[op2(stack_trace)] pub fn op_fs_symlink_sync

( state: &mut OpState, #[string] oldpath: &str, #[string] newpath: &str, #[serde] file_type: Option, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -695,13 +762,13 @@ where Ok(()) } -#[op2(async)] +#[op2(async, stack_trace)] pub async fn op_fs_symlink_async

( state: Rc>, #[string] oldpath: String, #[string] newpath: String, #[serde] file_type: Option, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -723,12 +790,12 @@ where Ok(()) } -#[op2] +#[op2(stack_trace)] #[string] pub fn op_fs_read_link_sync

( state: &mut OpState, #[string] path: String, -) -> Result +) -> Result where P: FsPermissions + 'static, { @@ -743,12 +810,12 @@ where Ok(target_string) } -#[op2(async)] +#[op2(async, stack_trace)] #[string] pub async fn op_fs_read_link_async

( state: Rc>, #[string] path: String, -) -> Result +) -> Result where P: FsPermissions + 'static, { @@ -768,12 +835,12 @@ where Ok(target_string) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_fs_truncate_sync

( state: &mut OpState, #[string] path: &str, #[number] len: u64, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -788,12 +855,12 @@ where Ok(()) } -#[op2(async)] +#[op2(async, stack_trace)] pub async fn op_fs_truncate_async

( state: Rc>, #[string] path: String, #[number] len: u64, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -812,7 +879,7 @@ where Ok(()) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_fs_utime_sync

( state: &mut OpState, #[string] path: &str, @@ -820,7 +887,7 @@ pub fn op_fs_utime_sync

( #[smi] atime_nanos: u32, #[number] mtime_secs: i64, #[smi] mtime_nanos: u32, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -833,7 +900,7 @@ where Ok(()) } -#[op2(async)] +#[op2(async, stack_trace)] pub async fn op_fs_utime_async

( state: Rc>, #[string] path: String, @@ -841,7 +908,7 @@ pub async fn op_fs_utime_async

( #[smi] atime_nanos: u32, #[number] mtime_secs: i64, #[smi] mtime_nanos: u32, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -864,14 +931,14 @@ where Ok(()) } -#[op2] +#[op2(stack_trace)] #[string] pub fn op_fs_make_temp_dir_sync

( state: &mut OpState, #[string] dir_arg: Option, #[string] prefix: Option, #[string] suffix: Option, -) -> Result +) -> Result where P: FsPermissions + 'static, { @@ -886,7 +953,7 @@ where const MAX_TRIES: u32 = 10; for _ in 0..MAX_TRIES { let path = tmp_name(&mut rng, &dir, prefix.as_deref(), suffix.as_deref())?; - match fs.mkdir_sync(&path, false, 0o700) { + match fs.mkdir_sync(&path, false, Some(0o700)) { Ok(_) => { // PERMISSIONS: ensure the absolute path is not leaked let path = strip_dir_prefix(&dir, dir_arg.as_deref(), path)?; @@ -906,14 +973,14 @@ where .context("tmpdir") } -#[op2(async)] +#[op2(async, stack_trace)] #[string] pub async fn op_fs_make_temp_dir_async

( state: Rc>, #[string] dir_arg: Option, #[string] prefix: Option, #[string] suffix: Option, -) -> Result +) -> Result where P: FsPermissions + 'static, { @@ -928,7 +995,11 @@ where const MAX_TRIES: u32 = 10; for _ in 0..MAX_TRIES { let path = tmp_name(&mut rng, &dir, prefix.as_deref(), suffix.as_deref())?; - match fs.clone().mkdir_async(path.clone(), false, 0o700).await { + match fs + .clone() + .mkdir_async(path.clone(), false, Some(0o700)) + .await + { Ok(_) => { // PERMISSIONS: ensure the absolute path is not leaked let path = strip_dir_prefix(&dir, dir_arg.as_deref(), path)?; @@ -948,14 +1019,14 @@ where .context("tmpdir") } -#[op2] +#[op2(stack_trace)] #[string] pub fn op_fs_make_temp_file_sync

( state: &mut OpState, #[string] dir_arg: Option, #[string] prefix: Option, #[string] suffix: Option, -) -> Result +) -> Result where P: FsPermissions + 'static, { @@ -996,14 +1067,14 @@ where .context("tmpfile") } -#[op2(async)] +#[op2(async, stack_trace)] #[string] pub async fn op_fs_make_temp_file_async

( state: Rc>, #[string] dir_arg: Option, #[string] prefix: Option, #[string] suffix: Option, -) -> Result +) -> Result where P: FsPermissions + 'static, { @@ -1065,7 +1136,7 @@ fn make_temp_check_sync

( state: &mut OpState, dir: Option<&str>, api_name: &str, -) -> Result<(PathBuf, FileSystemRc), AnyError> +) -> Result<(PathBuf, FileSystemRc), FsOpsError> where P: FsPermissions + 'static, { @@ -1087,7 +1158,7 @@ fn make_temp_check_async

( state: Rc>, dir: Option<&str>, api_name: &str, -) -> Result<(PathBuf, FileSystemRc), AnyError> +) -> Result<(PathBuf, FileSystemRc), FsOpsError> where P: FsPermissions + 'static, { @@ -1112,10 +1183,12 @@ where fn validate_temporary_filename_component( component: &str, #[allow(unused_variables)] suffix: bool, -) -> Result<(), AnyError> { +) -> Result<(), FsOpsError> { // Ban ASCII and Unicode control characters: these will often fail if let Some(c) = component.matches(|c: char| c.is_control()).next() { - bail!("Invalid control character in prefix or suffix: {:?}", c); + return Err( + FsOpsErrorKind::InvalidControlCharacter(c.to_string()).into_box(), + ); } // Windows has the most restrictive filenames. As temp files aren't normal files, we just // use this set of banned characters for all platforms because wildcard-like files can also @@ -1131,13 +1204,13 @@ fn validate_temporary_filename_component( .matches(|c: char| "<>:\"/\\|?*".contains(c)) .next() { - bail!("Invalid character in prefix or suffix: {:?}", c); + return Err(FsOpsErrorKind::InvalidCharacter(c.to_string()).into_box()); } // This check is only for Windows #[cfg(windows)] if suffix && component.ends_with(|c: char| ". ".contains(c)) { - bail!("Invalid trailing character in suffix"); + return Err(FsOpsErrorKind::InvalidTrailingCharacter.into_box()); } Ok(()) @@ -1148,7 +1221,7 @@ fn tmp_name( dir: &Path, prefix: Option<&str>, suffix: Option<&str>, -) -> Result { +) -> Result { let prefix = prefix.unwrap_or(""); validate_temporary_filename_component(prefix, false)?; let suffix = suffix.unwrap_or(""); @@ -1166,7 +1239,7 @@ fn tmp_name( Ok(path) } -#[op2] +#[op2(stack_trace)] pub fn op_fs_write_file_sync

( state: &mut OpState, #[string] path: String, @@ -1175,7 +1248,7 @@ pub fn op_fs_write_file_sync

( create: bool, create_new: bool, #[buffer] data: JsBuffer, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -1192,7 +1265,7 @@ where Ok(()) } -#[op2(async)] +#[op2(async, stack_trace)] #[allow(clippy::too_many_arguments)] pub async fn op_fs_write_file_async

( state: Rc>, @@ -1203,7 +1276,7 @@ pub async fn op_fs_write_file_async

( create_new: bool, #[buffer] data: JsBuffer, #[smi] cancel_rid: Option, -) -> Result<(), AnyError> +) -> Result<(), FsOpsError> where P: FsPermissions + 'static, { @@ -1246,12 +1319,12 @@ where Ok(()) } -#[op2] +#[op2(stack_trace)] #[serde] pub fn op_fs_read_file_sync

( state: &mut OpState, #[string] path: String, -) -> Result +) -> Result where P: FsPermissions + 'static, { @@ -1264,16 +1337,17 @@ where .read_file_sync(&path, Some(&mut access_check)) .map_err(|error| map_permission_error("readfile", error, &path))?; - Ok(buf.into()) + // todo(https://github.com/denoland/deno/issues/27107): do not clone here + Ok(buf.into_owned().into_boxed_slice().into()) } -#[op2(async)] +#[op2(async, stack_trace)] #[serde] pub async fn op_fs_read_file_async

( state: Rc>, #[string] path: String, #[smi] cancel_rid: Option, -) -> Result +) -> Result where P: FsPermissions + 'static, { @@ -1306,15 +1380,61 @@ where .map_err(|error| map_permission_error("readfile", error, &path))? }; - Ok(buf.into()) + // todo(https://github.com/denoland/deno/issues/27107): do not clone here + Ok(buf.into_owned().into_boxed_slice().into()) } -#[op2] -#[string] +// todo(https://github.com/denoland/deno_core/pull/986): remove +// when upgrading deno_core +#[derive(Debug)] +pub struct FastStringV8AllocationError; + +impl std::error::Error for FastStringV8AllocationError {} + +impl std::fmt::Display for FastStringV8AllocationError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "failed to allocate string; buffer exceeds maximum length" + ) + } +} + +/// Maintains a static reference to the string if possible. +pub struct V8MaybeStaticStr(pub Cow<'static, str>); + +impl<'s> ToV8<'s> for V8MaybeStaticStr { + type Error = FastStringV8AllocationError; + + #[inline] + fn to_v8( + self, + scope: &mut v8::HandleScope<'s>, + ) -> Result, Self::Error> { + // todo(https://github.com/denoland/deno_core/pull/986): remove this check + // when upgrading deno_core + const MAX_V8_STRING_LENGTH: usize = 536870888; + if self.0.len() > MAX_V8_STRING_LENGTH { + return Err(FastStringV8AllocationError); + } + + Ok( + match self.0 { + Cow::Borrowed(text) => FastString::from_static(text), + Cow::Owned(value) => value.into(), + } + .v8_string(scope) + .into(), + ) + } +} + +#[op2(stack_trace)] +#[to_v8] pub fn op_fs_read_file_text_sync

( state: &mut OpState, #[string] path: String, -) -> Result +) -> Result where P: FsPermissions + 'static, { @@ -1326,17 +1446,16 @@ where let str = fs .read_text_file_lossy_sync(&path, Some(&mut access_check)) .map_err(|error| map_permission_error("readfile", error, &path))?; - - Ok(str) + Ok(V8MaybeStaticStr(str)) } -#[op2(async)] -#[string] +#[op2(async, stack_trace)] +#[to_v8] pub async fn op_fs_read_file_text_async

( state: Rc>, #[string] path: String, #[smi] cancel_rid: Option, -) -> Result +) -> Result where P: FsPermissions + 'static, { @@ -1370,16 +1489,16 @@ where .map_err(|error| map_permission_error("readfile", error, &path))? }; - Ok(str) + Ok(V8MaybeStaticStr(str)) } -fn to_seek_from(offset: i64, whence: i32) -> Result { +fn to_seek_from(offset: i64, whence: i32) -> Result { let seek_from = match whence { 0 => SeekFrom::Start(offset as u64), 1 => SeekFrom::Current(offset), 2 => SeekFrom::End(offset), _ => { - return Err(type_error(format!("Invalid seek mode: {whence}"))); + return Err(FsOpsErrorKind::InvalidSeekMode(whence).into_box()); } }; Ok(seek_from) @@ -1392,9 +1511,10 @@ pub fn op_fs_seek_sync( #[smi] rid: ResourceId, #[number] offset: i64, #[smi] whence: i32, -) -> Result { +) -> Result { let pos = to_seek_from(offset, whence)?; - let file = FileResource::get_file(state, rid)?; + let file = + FileResource::get_file(state, rid).map_err(FsOpsErrorKind::Resource)?; let cursor = file.seek_sync(pos)?; Ok(cursor) } @@ -1406,9 +1526,10 @@ pub async fn op_fs_seek_async( #[smi] rid: ResourceId, #[number] offset: i64, #[smi] whence: i32, -) -> Result { +) -> Result { let pos = to_seek_from(offset, whence)?; - let file = FileResource::get_file(&state.borrow(), rid)?; + let file = FileResource::get_file(&state.borrow(), rid) + .map_err(FsOpsErrorKind::Resource)?; let cursor = file.seek_async(pos).await?; Ok(cursor) } @@ -1417,8 +1538,9 @@ pub async fn op_fs_seek_async( pub fn op_fs_file_sync_data_sync( state: &mut OpState, #[smi] rid: ResourceId, -) -> Result<(), AnyError> { - let file = FileResource::get_file(state, rid)?; +) -> Result<(), FsOpsError> { + let file = + FileResource::get_file(state, rid).map_err(FsOpsErrorKind::Resource)?; file.datasync_sync()?; Ok(()) } @@ -1427,8 +1549,9 @@ pub fn op_fs_file_sync_data_sync( pub async fn op_fs_file_sync_data_async( state: Rc>, #[smi] rid: ResourceId, -) -> Result<(), AnyError> { - let file = FileResource::get_file(&state.borrow(), rid)?; +) -> Result<(), FsOpsError> { + let file = FileResource::get_file(&state.borrow(), rid) + .map_err(FsOpsErrorKind::Resource)?; file.datasync_async().await?; Ok(()) } @@ -1437,8 +1560,9 @@ pub async fn op_fs_file_sync_data_async( pub fn op_fs_file_sync_sync( state: &mut OpState, #[smi] rid: ResourceId, -) -> Result<(), AnyError> { - let file = FileResource::get_file(state, rid)?; +) -> Result<(), FsOpsError> { + let file = + FileResource::get_file(state, rid).map_err(FsOpsErrorKind::Resource)?; file.sync_sync()?; Ok(()) } @@ -1447,8 +1571,9 @@ pub fn op_fs_file_sync_sync( pub async fn op_fs_file_sync_async( state: Rc>, #[smi] rid: ResourceId, -) -> Result<(), AnyError> { - let file = FileResource::get_file(&state.borrow(), rid)?; +) -> Result<(), FsOpsError> { + let file = FileResource::get_file(&state.borrow(), rid) + .map_err(FsOpsErrorKind::Resource)?; file.sync_async().await?; Ok(()) } @@ -1458,8 +1583,9 @@ pub fn op_fs_file_stat_sync( state: &mut OpState, #[smi] rid: ResourceId, #[buffer] stat_out_buf: &mut [u32], -) -> Result<(), AnyError> { - let file = FileResource::get_file(state, rid)?; +) -> Result<(), FsOpsError> { + let file = + FileResource::get_file(state, rid).map_err(FsOpsErrorKind::Resource)?; let stat = file.stat_sync()?; let serializable_stat = SerializableStat::from(stat); serializable_stat.write(stat_out_buf); @@ -1471,8 +1597,9 @@ pub fn op_fs_file_stat_sync( pub async fn op_fs_file_stat_async( state: Rc>, #[smi] rid: ResourceId, -) -> Result { - let file = FileResource::get_file(&state.borrow(), rid)?; +) -> Result { + let file = FileResource::get_file(&state.borrow(), rid) + .map_err(FsOpsErrorKind::Resource)?; let stat = file.stat_async().await?; Ok(stat.into()) } @@ -1482,8 +1609,9 @@ pub fn op_fs_flock_sync( state: &mut OpState, #[smi] rid: ResourceId, exclusive: bool, -) -> Result<(), AnyError> { - let file = FileResource::get_file(state, rid)?; +) -> Result<(), FsOpsError> { + let file = + FileResource::get_file(state, rid).map_err(FsOpsErrorKind::Resource)?; file.lock_sync(exclusive)?; Ok(()) } @@ -1493,8 +1621,9 @@ pub async fn op_fs_flock_async( state: Rc>, #[smi] rid: ResourceId, exclusive: bool, -) -> Result<(), AnyError> { - let file = FileResource::get_file(&state.borrow(), rid)?; +) -> Result<(), FsOpsError> { + let file = FileResource::get_file(&state.borrow(), rid) + .map_err(FsOpsErrorKind::Resource)?; file.lock_async(exclusive).await?; Ok(()) } @@ -1503,8 +1632,9 @@ pub async fn op_fs_flock_async( pub fn op_fs_funlock_sync( state: &mut OpState, #[smi] rid: ResourceId, -) -> Result<(), AnyError> { - let file = FileResource::get_file(state, rid)?; +) -> Result<(), FsOpsError> { + let file = + FileResource::get_file(state, rid).map_err(FsOpsErrorKind::Resource)?; file.unlock_sync()?; Ok(()) } @@ -1513,8 +1643,9 @@ pub fn op_fs_funlock_sync( pub async fn op_fs_funlock_async( state: Rc>, #[smi] rid: ResourceId, -) -> Result<(), AnyError> { - let file = FileResource::get_file(&state.borrow(), rid)?; +) -> Result<(), FsOpsError> { + let file = FileResource::get_file(&state.borrow(), rid) + .map_err(FsOpsErrorKind::Resource)?; file.unlock_async().await?; Ok(()) } @@ -1524,8 +1655,9 @@ pub fn op_fs_ftruncate_sync( state: &mut OpState, #[smi] rid: ResourceId, #[number] len: u64, -) -> Result<(), AnyError> { - let file = FileResource::get_file(state, rid)?; +) -> Result<(), FsOpsError> { + let file = + FileResource::get_file(state, rid).map_err(FsOpsErrorKind::Resource)?; file.truncate_sync(len)?; Ok(()) } @@ -1535,8 +1667,9 @@ pub async fn op_fs_file_truncate_async( state: Rc>, #[smi] rid: ResourceId, #[number] len: u64, -) -> Result<(), AnyError> { - let file = FileResource::get_file(&state.borrow(), rid)?; +) -> Result<(), FsOpsError> { + let file = FileResource::get_file(&state.borrow(), rid) + .map_err(FsOpsErrorKind::Resource)?; file.truncate_async(len).await?; Ok(()) } @@ -1549,8 +1682,9 @@ pub fn op_fs_futime_sync( #[smi] atime_nanos: u32, #[number] mtime_secs: i64, #[smi] mtime_nanos: u32, -) -> Result<(), AnyError> { - let file = FileResource::get_file(state, rid)?; +) -> Result<(), FsOpsError> { + let file = + FileResource::get_file(state, rid).map_err(FsOpsErrorKind::Resource)?; file.utime_sync(atime_secs, atime_nanos, mtime_secs, mtime_nanos)?; Ok(()) } @@ -1563,42 +1697,64 @@ pub async fn op_fs_futime_async( #[smi] atime_nanos: u32, #[number] mtime_secs: i64, #[smi] mtime_nanos: u32, -) -> Result<(), AnyError> { - let file = FileResource::get_file(&state.borrow(), rid)?; +) -> Result<(), FsOpsError> { + let file = FileResource::get_file(&state.borrow(), rid) + .map_err(FsOpsErrorKind::Resource)?; file .utime_async(atime_secs, atime_nanos, mtime_secs, mtime_nanos) .await?; Ok(()) } -trait WithContext { - fn context>>( - self, - desc: E, - ) -> AnyError; +#[derive(Debug)] +pub struct OperationError { + operation: &'static str, + kind: OperationErrorKind, + pub err: FsError, } -impl WithContext for FsError { - fn context>>( - self, - desc: E, - ) -> AnyError { - match self { - FsError::Io(io) => { - AnyError::new(io::Error::new(io.kind(), desc)).context(io) +impl std::fmt::Display for OperationError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + if let FsError::Io(e) = &self.err { + std::fmt::Display::fmt(&e, f)?; + f.write_str(": ")?; + } + + f.write_str(self.operation)?; + + match &self.kind { + OperationErrorKind::Bare => Ok(()), + OperationErrorKind::WithPath(path) => write!(f, " '{}'", path.display()), + OperationErrorKind::WithTwoPaths(from, to) => { + write!(f, " '{}' -> '{}'", from.display(), to.display()) } - _ => self.into(), } } } +impl std::error::Error for OperationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + if let FsError::Io(err) = &self.err { + Some(err) + } else { + None + } + } +} + +#[derive(Debug)] +pub enum OperationErrorKind { + Bare, + WithPath(PathBuf), + WithTwoPaths(PathBuf, PathBuf), +} + trait MapErrContext { type R; - fn context_fn(self, f: F) -> Self::R + fn context_fn(self, f: F) -> Self::R where - F: FnOnce() -> E, - E: Into>; + F: FnOnce(FsError) -> OperationError; fn context(self, desc: &'static str) -> Self::R; @@ -1613,25 +1769,29 @@ trait MapErrContext { } impl MapErrContext for Result { - type R = Result; + type R = Result; - fn context_fn(self, f: F) -> Self::R + fn context_fn(self, f: F) -> Self::R where - F: FnOnce() -> E, - E: Into>, + F: FnOnce(FsError) -> OperationError, { - self.map_err(|err| { - let message = f(); - err.context(message) - }) + self.map_err(|err| FsOpsErrorKind::OperationError(f(err)).into_box()) } - fn context(self, desc: &'static str) -> Self::R { - self.context_fn(move || desc) + fn context(self, operation: &'static str) -> Self::R { + self.context_fn(move |err| OperationError { + operation, + kind: OperationErrorKind::Bare, + err, + }) } fn context_path(self, operation: &'static str, path: &Path) -> Self::R { - self.context_fn(|| format!("{operation} '{}'", path.display())) + self.context_fn(|err| OperationError { + operation, + kind: OperationErrorKind::WithPath(path.to_path_buf()), + err, + }) } fn context_two_path( @@ -1640,21 +1800,20 @@ impl MapErrContext for Result { oldpath: &Path, newpath: &Path, ) -> Self::R { - self.context_fn(|| { - format!( - "{operation} '{}' -> '{}'", - oldpath.display(), - newpath.display() - ) + self.context_fn(|err| OperationError { + operation, + kind: OperationErrorKind::WithTwoPaths( + oldpath.to_path_buf(), + newpath.to_path_buf(), + ), + err, }) } } -fn path_into_string(s: std::ffi::OsString) -> Result { - s.into_string().map_err(|s| { - let message = format!("File name or path {s:?} is not valid UTF-8"); - custom_error("InvalidData", message) - }) +fn path_into_string(s: std::ffi::OsString) -> Result { + s.into_string() + .map_err(|e| FsOpsErrorKind::InvalidUtf8(e).into_box()) } macro_rules! create_struct_writer { @@ -1695,6 +1854,8 @@ create_struct_writer! { atime: u64, birthtime_set: bool, birthtime: u64, + ctime_set: bool, + ctime: u64, // Following are only valid under Unix. dev: u64, ino: u64, @@ -1726,6 +1887,8 @@ impl From for SerializableStat { atime: stat.atime.unwrap_or(0), birthtime_set: stat.birthtime.is_some(), birthtime: stat.birthtime.unwrap_or(0), + ctime_set: stat.ctime.is_some(), + ctime: stat.ctime.unwrap_or(0), dev: stat.dev, ino: stat.ino, diff --git a/ext/fs/std_fs.rs b/ext/fs/std_fs.rs index d8d5f650279e68..86ad2131601bab 100644 --- a/ext/fs/std_fs.rs +++ b/ext/fs/std_fs.rs @@ -2,6 +2,7 @@ #![allow(clippy::disallowed_methods)] +use std::borrow::Cow; use std::env::current_dir; use std::fs; use std::io; @@ -11,13 +12,13 @@ use std::path::Path; use std::path::PathBuf; use std::rc::Rc; -use deno_core::normalize_path; use deno_core::unsync::spawn_blocking; use deno_io::fs::File; use deno_io::fs::FsError; use deno_io::fs::FsResult; use deno_io::fs::FsStat; use deno_io::StdFileResourceInner; +use deno_path_util::normalize_path; use crate::interface::AccessCheckCb; use crate::interface::FsDirEntry; @@ -101,7 +102,7 @@ impl FileSystem for RealFs { &self, path: &Path, recursive: bool, - mode: u32, + mode: Option, ) -> FsResult<()> { mkdir(path, recursive, mode) } @@ -109,7 +110,7 @@ impl FileSystem for RealFs { &self, path: PathBuf, recursive: bool, - mode: u32, + mode: Option, ) -> FsResult<()> { spawn_blocking(move || mkdir(&path, recursive, mode)).await? } @@ -371,7 +372,7 @@ impl FileSystem for RealFs { &self, path: &Path, access_check: Option, - ) -> FsResult> { + ) -> FsResult> { let mut file = open_with_access_check( OpenOptions { read: true, @@ -382,13 +383,13 @@ impl FileSystem for RealFs { )?; let mut buf = Vec::new(); file.read_to_end(&mut buf)?; - Ok(buf) + Ok(Cow::Owned(buf)) } async fn read_file_async<'a>( &'a self, path: PathBuf, access_check: Option>, - ) -> FsResult> { + ) -> FsResult> { let mut file = open_with_access_check( OpenOptions { read: true, @@ -400,18 +401,18 @@ impl FileSystem for RealFs { spawn_blocking(move || { let mut buf = Vec::new(); file.read_to_end(&mut buf)?; - Ok::<_, FsError>(buf) + Ok::<_, FsError>(Cow::Owned(buf)) }) .await? .map_err(Into::into) } } -fn mkdir(path: &Path, recursive: bool, mode: u32) -> FsResult<()> { +fn mkdir(path: &Path, recursive: bool, mode: Option) -> FsResult<()> { let mut builder = fs::DirBuilder::new(); builder.recursive(recursive); #[cfg(unix)] - { + if let Some(mode) = mode { use std::os::unix::fs::DirBuilderExt; builder.mode(mode); } @@ -821,24 +822,46 @@ fn stat_extra( Ok(info.dwVolumeSerialNumber as u64) } + const WINDOWS_TICK: i64 = 10_000; // 100-nanosecond intervals in a millisecond + const SEC_TO_UNIX_EPOCH: i64 = 11_644_473_600; // Seconds between Windows epoch and Unix epoch + + fn windows_time_to_unix_time_msec(windows_time: &i64) -> i64 { + let milliseconds_since_windows_epoch = windows_time / WINDOWS_TICK; + milliseconds_since_windows_epoch - SEC_TO_UNIX_EPOCH * 1000 + } + use windows_sys::Wdk::Storage::FileSystem::FILE_ALL_INFORMATION; + use windows_sys::Win32::Foundation::NTSTATUS; unsafe fn query_file_information( handle: winapi::shared::ntdef::HANDLE, - ) -> std::io::Result { + ) -> Result { use windows_sys::Wdk::Storage::FileSystem::NtQueryInformationFile; + use windows_sys::Win32::Foundation::RtlNtStatusToDosError; + use windows_sys::Win32::Foundation::ERROR_MORE_DATA; + use windows_sys::Win32::System::IO::IO_STATUS_BLOCK; let mut info = std::mem::MaybeUninit::::zeroed(); + let mut io_status_block = + std::mem::MaybeUninit::::zeroed(); let status = NtQueryInformationFile( handle as _, - std::ptr::null_mut(), + io_status_block.as_mut_ptr(), info.as_mut_ptr() as *mut _, std::mem::size_of::() as _, 18, /* FileAllInformation */ ); if status < 0 { - return Err(std::io::Error::last_os_error()); + let converted_status = RtlNtStatusToDosError(status); + + // If error more data is returned, then it means that the buffer is too small to get full filename information + // to have that we should retry. However, since we only use BasicInformation and StandardInformation, it is fine to ignore it + // since struct is populated with other data anyway. + // https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntifs/nf-ntifs-ntqueryinformationfile#remarksdd + if converted_status != ERROR_MORE_DATA { + return Err(converted_status as NTSTATUS); + } } Ok(info.assume_init()) @@ -862,10 +885,13 @@ fn stat_extra( } let result = get_dev(file_handle); - CloseHandle(file_handle); fsstat.dev = result?; if let Ok(file_info) = query_file_information(file_handle) { + fsstat.ctime = Some(windows_time_to_unix_time_msec( + &file_info.BasicInformation.ChangeTime, + ) as u64); + if file_info.BasicInformation.FileAttributes & winapi::um::winnt::FILE_ATTRIBUTE_REPARSE_POINT != 0 @@ -898,6 +924,7 @@ fn stat_extra( } } + CloseHandle(file_handle); Ok(()) } } @@ -929,7 +956,7 @@ fn exists(path: &Path) -> bool { } fn realpath(path: &Path) -> FsResult { - Ok(deno_core::strip_unc_prefix(path.canonicalize()?)) + Ok(deno_path_util::strip_unc_prefix(path.canonicalize()?)) } fn read_dir(path: &Path) -> FsResult> { diff --git a/ext/http/00_serve.ts b/ext/http/00_serve.ts index 3b9b085a2274e3..446533e91043f9 100644 --- a/ext/http/00_serve.ts +++ b/ext/http/00_serve.ts @@ -14,6 +14,7 @@ import { op_http_get_request_headers, op_http_get_request_method_and_url, op_http_read_request_body, + op_http_request_on_cancel, op_http_serve, op_http_serve_on, op_http_set_promise_complete, @@ -33,6 +34,7 @@ const { ObjectHasOwn, ObjectPrototypeIsPrototypeOf, PromisePrototypeCatch, + SafePromisePrototypeFinally, PromisePrototypeThen, StringPrototypeIncludes, Symbol, @@ -41,6 +43,10 @@ const { Uint8Array, Promise, } = primordials; +const { + getAsyncContext, + setAsyncContext, +} = core; import { InnerBody } from "ext:deno_fetch/22_body.js"; import { Event } from "ext:deno_web/02_event.js"; @@ -76,9 +82,23 @@ import { ReadableStreamPrototype, resourceForReadableStream, } from "ext:deno_web/06_streams.js"; -import { listen, listenOptionApiName, TcpConn } from "ext:deno_net/01_net.js"; +import { + listen, + listenOptionApiName, + UpgradedConn, +} from "ext:deno_net/01_net.js"; import { hasTlsKeyPairOptions, listenTls } from "ext:deno_net/02_tls.js"; import { SymbolAsyncDispose } from "ext:deno_web/00_infra.js"; +import { + endSpan, + enterSpan, + Span, + TRACING_ENABLED, +} from "ext:deno_telemetry/telemetry.ts"; +import { + updateSpanFromRequest, + updateSpanFromResponse, +} from "ext:deno_telemetry/util.ts"; const _upgraded = Symbol("_upgraded"); @@ -189,7 +209,7 @@ class InnerRequest { const upgradeRid = op_http_upgrade_raw(external); - const conn = new TcpConn( + const conn = new UpgradedConn( upgradeRid, underlyingConn?.remoteAddr, underlyingConn?.localAddr, @@ -369,6 +389,18 @@ class InnerRequest { get external() { return this.#external; } + + onCancel(callback) { + if (this.#external === null) { + callback(); + return; + } + + PromisePrototypeThen( + op_http_request_on_cancel(this.#external), + callback, + ); + } } class CallbackContext { @@ -380,8 +412,10 @@ class CallbackContext { /** @type {Promise | undefined} */ closing; listener; + asyncContext; constructor(signal, args, listener) { + this.asyncContext = getAsyncContext(); // The abort signal triggers a non-graceful shutdown signal?.addEventListener( "abort", @@ -490,7 +524,7 @@ function fastSyncResponseOrStream( * This function returns a promise that will only reject in the case of abnormal exit. */ function mapToCallback(context, callback, onError) { - return async function (req) { + let mapped = async function (req, span) { // Get the response from the user-provided callback. If that fails, use onError. If that fails, return a fallback // 500 error. let innerRequest; @@ -499,6 +533,11 @@ function mapToCallback(context, callback, onError) { innerRequest = new InnerRequest(req, context); const request = fromInnerRequest(innerRequest, "immutable"); innerRequest.request = request; + + if (span) { + updateSpanFromRequest(span, request); + } + response = await callback( request, new ServeHandlerInfo(innerRequest), @@ -536,6 +575,11 @@ function mapToCallback(context, callback, onError) { response = internalServerError(); } } + + if (span) { + updateSpanFromResponse(span, response); + } + const inner = toInnerResponse(response); if (innerRequest?.[_upgraded]) { // We're done here as the connection has been upgraded during the callback and no longer requires servicing. @@ -568,6 +612,38 @@ function mapToCallback(context, callback, onError) { fastSyncResponseOrStream(req, inner.body, status, innerRequest); }; + + if (TRACING_ENABLED) { + const origMapped = mapped; + mapped = function (req, _span) { + const oldCtx = getAsyncContext(); + setAsyncContext(context.asyncContext); + const span = new Span("deno.serve", { kind: 1 }); + try { + enterSpan(span); + return SafePromisePrototypeFinally( + origMapped(req, span), + () => endSpan(span), + ); + } finally { + // equiv to exitSpan. + setAsyncContext(oldCtx); + } + }; + } else { + const origMapped = mapped; + mapped = function (req, span) { + const oldCtx = getAsyncContext(); + setAsyncContext(context.asyncContext); + try { + return origMapped(req, span); + } finally { + setAsyncContext(oldCtx); + } + }; + } + + return mapped; } type RawHandler = ( @@ -765,7 +841,7 @@ function serveHttpOn(context, addr, callback) { // Attempt to pull as many requests out of the queue as possible before awaiting. This API is // a synchronous, non-blocking API that returns u32::MAX if anything goes wrong. while ((req = op_http_try_wait(rid)) !== null) { - PromisePrototypeCatch(callback(req), promiseErrorHandler); + PromisePrototypeCatch(callback(req, undefined), promiseErrorHandler); } currentPromise = op_http_wait(rid); if (!ref) { @@ -785,7 +861,7 @@ function serveHttpOn(context, addr, callback) { if (req === null) { break; } - PromisePrototypeCatch(callback(req), promiseErrorHandler); + PromisePrototypeCatch(callback(req, undefined), promiseErrorHandler); } try { diff --git a/ext/http/Cargo.toml b/ext/http/Cargo.toml index ba21e3f30dc3ba..5d7eec73531a09 100644 --- a/ext/http/Cargo.toml +++ b/ext/http/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_http" -version = "0.166.0" +version = "0.178.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/http/fly_accept_encoding.rs b/ext/http/fly_accept_encoding.rs index 94e33687670c63..4d6fd2231efb1a 100644 --- a/ext/http/fly_accept_encoding.rs +++ b/ext/http/fly_accept_encoding.rs @@ -119,7 +119,7 @@ fn encodings_iter_inner<'s>( }; Some(Ok((encoding, qval))) }) - .map(|r| r?) // flatten Result Result { +) -> Result { // SAFETY: external is deleted before calling this op. let http = unsafe { take_external!(external, "op_http_upgrade_raw") }; @@ -177,7 +197,7 @@ pub fn op_http_upgrade_raw( upgraded.write_all(&bytes).await?; break upgraded; } - Err(err) => return Err(err), + Err(err) => return Err(HttpNextError::WebSocketUpgrade(err)), } }; @@ -193,7 +213,7 @@ pub fn op_http_upgrade_raw( } read_tx.write_all(&buf[..read]).await?; } - Ok::<_, AnyError>(()) + Ok::<_, HttpNextError>(()) }); spawn(async move { let mut buf = [0; 1024]; @@ -204,7 +224,7 @@ pub fn op_http_upgrade_raw( } upgraded_tx.write_all(&buf[..read]).await?; } - Ok::<_, AnyError>(()) + Ok::<_, HttpNextError>(()) }); Ok(()) @@ -223,7 +243,7 @@ pub async fn op_http_upgrade_websocket_next( state: Rc>, external: *const c_void, #[serde] headers: Vec<(ByteString, ByteString)>, -) -> Result { +) -> Result { let http = // SAFETY: external is deleted before calling this op. unsafe { take_external!(external, "op_http_upgrade_websocket_next") }; @@ -246,7 +266,11 @@ pub async fn op_http_upgrade_websocket_next( // Stage 3: take the extracted raw network stream and upgrade it to a websocket, then return it let (stream, bytes) = extract_network_stream(upgraded); - ws_create_server_stream(&mut state.borrow_mut(), stream, bytes) + Ok(ws_create_server_stream( + &mut state.borrow_mut(), + stream, + bytes, + )) } #[op2(fast)] @@ -296,7 +320,7 @@ where let authority: v8::Local = match request_properties.authority { Some(authority) => v8::String::new_from_utf8( scope, - authority.as_ref(), + authority.as_bytes(), v8::NewStringType::Normal, ) .unwrap() @@ -305,15 +329,25 @@ where }; // Only extract the path part - we handle authority elsewhere - let path = match &request_parts.uri.path_and_query() { - Some(path_and_query) => path_and_query.to_string(), - None => "".to_owned(), + let path = match request_parts.uri.path_and_query() { + Some(path_and_query) => { + let path = path_and_query.as_str(); + if matches!(path.as_bytes().first(), Some(b'/' | b'*')) { + Cow::Borrowed(path) + } else { + Cow::Owned(format!("/{}", path)) + } + } + None => Cow::Borrowed(""), }; - let path: v8::Local = - v8::String::new_from_utf8(scope, path.as_ref(), v8::NewStringType::Normal) - .unwrap() - .into(); + let path: v8::Local = v8::String::new_from_utf8( + scope, + path.as_bytes(), + v8::NewStringType::Normal, + ) + .unwrap() + .into(); let peer_address: v8::Local = v8::String::new_from_utf8( scope, @@ -531,6 +565,7 @@ fn is_request_compressible( match accept_encoding.to_str() { // Firefox and Chrome send this -- no need to parse Ok("gzip, deflate, br") => return Compression::Brotli, + Ok("gzip, deflate, br, zstd") => return Compression::Brotli, Ok("gzip") => return Compression::GZip, Ok("br") => return Compression::Brotli, _ => (), @@ -667,6 +702,27 @@ fn set_response( http.complete(); } +#[op2(fast)] +pub fn op_http_get_request_cancelled(external: *const c_void) -> bool { + let http = + // SAFETY: op is called with external. + unsafe { clone_external!(external, "op_http_get_request_cancelled") }; + http.cancelled() +} + +#[op2(async)] +pub async fn op_http_request_on_cancel(external: *const c_void) { + let http = + // SAFETY: op is called with external. + unsafe { clone_external!(external, "op_http_request_on_cancel") }; + let (tx, rx) = tokio::sync::oneshot::channel(); + + http.on_cancel(tx); + drop(http); + + rx.await.ok(); +} + /// Returned promise resolves when body streaming finishes. /// Call [`op_http_close_after_finish`] when done with the external. #[op2(async)] @@ -676,7 +732,7 @@ pub async fn op_http_set_response_body_resource( #[smi] stream_rid: ResourceId, auto_close: bool, status: u16, -) -> Result { +) -> Result { let http = // SAFETY: op is called with external. unsafe { clone_external!(external, "op_http_set_response_body_resource") }; @@ -691,9 +747,15 @@ pub async fn op_http_set_response_body_resource( let resource = { let mut state = state.borrow_mut(); if auto_close { - state.resource_table.take_any(stream_rid)? + state + .resource_table + .take_any(stream_rid) + .map_err(HttpNextError::Resource)? } else { - state.resource_table.get_any(stream_rid)? + state + .resource_table + .get_any(stream_rid) + .map_err(HttpNextError::Resource)? } }; @@ -760,10 +822,16 @@ fn serve_http11_unconditional( io: impl HttpServeStream, svc: impl HttpService + 'static, cancel: Rc, + http1_builder_hook: Option http1::Builder>, ) -> impl Future> + 'static { - let conn = http1::Builder::new() - .keep_alive(true) - .writev(*USE_WRITEV) + let mut builder = http1::Builder::new(); + builder.keep_alive(true).writev(*USE_WRITEV); + + if let Some(http1_builder_hook) = http1_builder_hook { + builder = http1_builder_hook(builder); + } + + let conn = builder .serve_connection(TokioIo::new(io), svc) .with_upgrades(); @@ -782,9 +850,17 @@ fn serve_http2_unconditional( io: impl HttpServeStream, svc: impl HttpService + 'static, cancel: Rc, + http2_builder_hook: Option< + fn(http2::Builder) -> http2::Builder, + >, ) -> impl Future> + 'static { - let conn = - http2::Builder::new(LocalExecutor).serve_connection(TokioIo::new(io), svc); + let mut builder = http2::Builder::new(LocalExecutor); + + if let Some(http2_builder_hook) = http2_builder_hook { + builder = http2_builder_hook(builder); + } + + let conn = builder.serve_connection(TokioIo::new(io), svc); async { match conn.or_abort(cancel).await { Err(mut conn) => { @@ -800,17 +876,18 @@ async fn serve_http2_autodetect( io: impl HttpServeStream, svc: impl HttpService + 'static, cancel: Rc, -) -> Result<(), AnyError> { + options: Options, +) -> Result<(), HttpNextError> { let prefix = NetworkStreamPrefixCheck::new(io, HTTP2_PREFIX); let (matches, io) = prefix.match_prefix().await?; if matches { - serve_http2_unconditional(io, svc, cancel) + serve_http2_unconditional(io, svc, cancel, options.http2_builder_hook) .await - .map_err(|e| e.into()) + .map_err(HttpNextError::Hyper) } else { - serve_http11_unconditional(io, svc, cancel) + serve_http11_unconditional(io, svc, cancel, options.http1_builder_hook) .await - .map_err(|e| e.into()) + .map_err(HttpNextError::Hyper) } } @@ -819,7 +896,8 @@ fn serve_https( request_info: HttpConnectionProperties, lifetime: HttpLifetime, tx: tokio::sync::mpsc::Sender>, -) -> JoinHandle> { + options: Options, +) -> JoinHandle> { let HttpLifetime { server_state, connection_cancel_handle, @@ -830,21 +908,31 @@ fn serve_https( handle_request(req, request_info.clone(), server_state.clone(), tx.clone()) }); spawn( - async { + async move { let handshake = io.handshake().await?; // If the client specifically negotiates a protocol, we will use it. If not, we'll auto-detect // based on the prefix bytes let handshake = handshake.alpn; if Some(TLS_ALPN_HTTP_2) == handshake.as_deref() { - serve_http2_unconditional(io, svc, listen_cancel_handle) - .await - .map_err(|e| e.into()) + serve_http2_unconditional( + io, + svc, + listen_cancel_handle, + options.http2_builder_hook, + ) + .await + .map_err(HttpNextError::Hyper) } else if Some(TLS_ALPN_HTTP_11) == handshake.as_deref() { - serve_http11_unconditional(io, svc, listen_cancel_handle) - .await - .map_err(|e| e.into()) + serve_http11_unconditional( + io, + svc, + listen_cancel_handle, + options.http1_builder_hook, + ) + .await + .map_err(HttpNextError::Hyper) } else { - serve_http2_autodetect(io, svc, listen_cancel_handle).await + serve_http2_autodetect(io, svc, listen_cancel_handle, options).await } } .try_or_cancel(connection_cancel_handle), @@ -856,7 +944,8 @@ fn serve_http( request_info: HttpConnectionProperties, lifetime: HttpLifetime, tx: tokio::sync::mpsc::Sender>, -) -> JoinHandle> { + options: Options, +) -> JoinHandle> { let HttpLifetime { server_state, connection_cancel_handle, @@ -867,7 +956,7 @@ fn serve_http( handle_request(req, request_info.clone(), server_state.clone(), tx.clone()) }); spawn( - serve_http2_autodetect(io, svc, listen_cancel_handle) + serve_http2_autodetect(io, svc, listen_cancel_handle, options) .try_or_cancel(connection_cancel_handle), ) } @@ -877,7 +966,8 @@ fn serve_http_on( listen_properties: &HttpListenProperties, lifetime: HttpLifetime, tx: tokio::sync::mpsc::Sender>, -) -> JoinHandle> + options: Options, +) -> JoinHandle> where HTTP: HttpPropertyExtractor, { @@ -888,14 +978,14 @@ where match network_stream { NetworkStream::Tcp(conn) => { - serve_http(conn, connection_properties, lifetime, tx) + serve_http(conn, connection_properties, lifetime, tx, options) } NetworkStream::Tls(conn) => { - serve_https(conn, connection_properties, lifetime, tx) + serve_https(conn, connection_properties, lifetime, tx, options) } #[cfg(unix)] NetworkStream::Unix(conn) => { - serve_http(conn, connection_properties, lifetime, tx) + serve_http(conn, connection_properties, lifetime, tx, options) } } } @@ -908,7 +998,7 @@ struct HttpLifetime { } struct HttpJoinHandle { - join_handle: AsyncRefCell>>>, + join_handle: AsyncRefCell>>>, connection_cancel_handle: Rc, listen_cancel_handle: Rc, rx: AsyncRefCell>>, @@ -968,12 +1058,13 @@ impl Drop for HttpJoinHandle { pub fn op_http_serve( state: Rc>, #[smi] listener_rid: ResourceId, -) -> Result<(ResourceId, &'static str, String), AnyError> +) -> Result<(ResourceId, &'static str, String), HttpNextError> where HTTP: HttpPropertyExtractor, { let listener = - HTTP::get_listener_for_rid(&mut state.borrow_mut(), listener_rid)?; + HTTP::get_listener_for_rid(&mut state.borrow_mut(), listener_rid) + .map_err(HttpNextError::Resource)?; let listen_properties = HTTP::listen_properties_from_listener(&listener)?; @@ -983,21 +1074,28 @@ where let lifetime = resource.lifetime(); + let options = { + let state = state.borrow(); + *state.borrow::() + }; + let listen_properties_clone: HttpListenProperties = listen_properties.clone(); let handle = spawn(async move { loop { let conn = HTTP::accept_connection_from_listener(&listener) .try_or_cancel(listen_cancel_clone.clone()) - .await?; + .await + .map_err(HttpNextError::HttpPropertyExtractor)?; serve_http_on::( conn, &listen_properties_clone, lifetime.clone(), tx.clone(), + options, ); } #[allow(unreachable_code)] - Ok::<_, AnyError>(()) + Ok::<_, HttpNextError>(()) }); // Set the handle after we start the future @@ -1017,25 +1115,31 @@ where pub fn op_http_serve_on( state: Rc>, #[smi] connection_rid: ResourceId, -) -> Result<(ResourceId, &'static str, String), AnyError> +) -> Result<(ResourceId, &'static str, String), HttpNextError> where HTTP: HttpPropertyExtractor, { let connection = - HTTP::get_connection_for_rid(&mut state.borrow_mut(), connection_rid)?; + HTTP::get_connection_for_rid(&mut state.borrow_mut(), connection_rid) + .map_err(HttpNextError::Resource)?; let listen_properties = HTTP::listen_properties_from_connection(&connection)?; let (tx, rx) = tokio::sync::mpsc::channel(10); let resource: Rc = Rc::new(HttpJoinHandle::new(rx)); - let handle: JoinHandle> = - serve_http_on::( - connection, - &listen_properties, - resource.lifetime(), - tx, - ); + let options = { + let state = state.borrow(); + *state.borrow::() + }; + + let handle = serve_http_on::( + connection, + &listen_properties, + resource.lifetime(), + tx, + options, + ); // Set the handle after we start the future *RcRef::map(&resource, |this| &this.join_handle) @@ -1081,12 +1185,13 @@ pub fn op_http_try_wait( pub async fn op_http_wait( state: Rc>, #[smi] rid: ResourceId, -) -> Result<*const c_void, AnyError> { +) -> Result<*const c_void, HttpNextError> { // We will get the join handle initially, as we might be consuming requests still let join_handle = state .borrow_mut() .resource_table - .get::(rid)?; + .get::(rid) + .map_err(HttpNextError::Resource)?; let cancel = join_handle.listen_cancel_handle(); let next = async { @@ -1113,13 +1218,12 @@ pub async fn op_http_wait( // Filter out shutdown (ENOTCONN) errors if let Err(err) = res { - if let Some(err) = err.source() { - if let Some(err) = err.downcast_ref::() { - if err.kind() == io::ErrorKind::NotConnected { - return Ok(null()); - } + if let HttpNextError::Io(err) = &err { + if err.kind() == io::ErrorKind::NotConnected { + return Ok(null()); } } + return Err(err); } @@ -1132,7 +1236,7 @@ pub fn op_http_cancel( state: &mut OpState, #[smi] rid: ResourceId, graceful: bool, -) -> Result<(), AnyError> { +) -> Result<(), deno_core::error::AnyError> { let join_handle = state.resource_table.get::(rid)?; if graceful { @@ -1152,11 +1256,12 @@ pub async fn op_http_close( state: Rc>, #[smi] rid: ResourceId, graceful: bool, -) -> Result<(), AnyError> { +) -> Result<(), HttpNextError> { let join_handle = state .borrow_mut() .resource_table - .take::(rid)?; + .take::(rid) + .map_err(HttpNextError::Resource)?; if graceful { http_general_trace!("graceful shutdown"); @@ -1202,23 +1307,26 @@ impl UpgradeStream { } } - async fn read(self: Rc, buf: &mut [u8]) -> Result { + async fn read( + self: Rc, + buf: &mut [u8], + ) -> Result { let cancel_handle = RcRef::map(self.clone(), |this| &this.cancel_handle); async { let read = RcRef::map(self, |this| &this.read); let mut read = read.borrow_mut().await; - Ok(Pin::new(&mut *read).read(buf).await?) + Pin::new(&mut *read).read(buf).await } .try_or_cancel(cancel_handle) .await } - async fn write(self: Rc, buf: &[u8]) -> Result { + async fn write(self: Rc, buf: &[u8]) -> Result { let cancel_handle = RcRef::map(self.clone(), |this| &this.cancel_handle); async { let write = RcRef::map(self, |this| &this.write); let mut write = write.borrow_mut().await; - Ok(Pin::new(&mut *write).write(buf).await?) + Pin::new(&mut *write).write(buf).await } .try_or_cancel(cancel_handle) .await @@ -1228,7 +1336,7 @@ impl UpgradeStream { self: Rc, buf1: &[u8], buf2: &[u8], - ) -> Result { + ) -> Result { let mut wr = RcRef::map(self, |r| &r.write).borrow_mut().await; let total = buf1.len() + buf2.len(); @@ -1281,9 +1389,12 @@ pub async fn op_raw_write_vectored( #[smi] rid: ResourceId, #[buffer] buf1: JsBuffer, #[buffer] buf2: JsBuffer, -) -> Result { - let resource: Rc = - state.borrow().resource_table.get::(rid)?; +) -> Result { + let resource: Rc = state + .borrow() + .resource_table + .get::(rid) + .map_err(HttpNextError::Resource)?; let nwritten = resource.write_vectored(&buf1, &buf2).await?; Ok(nwritten) } diff --git a/ext/http/lib.rs b/ext/http/lib.rs index 934f8a0024ae97..39b0bbc2af9691 100644 --- a/ext/http/lib.rs +++ b/ext/http/lib.rs @@ -6,8 +6,6 @@ use async_compression::Level; use base64::prelude::BASE64_STANDARD; use base64::Engine; use cache_control::CacheControl; -use deno_core::error::custom_error; -use deno_core::error::AnyError; use deno_core::futures::channel::mpsc; use deno_core::futures::channel::oneshot; use deno_core::futures::future::pending; @@ -41,6 +39,8 @@ use deno_net::raw::NetworkStream; use deno_websocket::ws_create_server_stream; use flate2::write::GzEncoder; use flate2::Compression; +use hyper::server::conn::http1; +use hyper::server::conn::http2; use hyper_util::rt::TokioIo; use hyper_v014::body::Bytes; use hyper_v014::body::HttpBody; @@ -89,11 +89,33 @@ mod service; mod websocket_upgrade; use fly_accept_encoding::Encoding; +pub use http_next::HttpNextError; pub use request_properties::DefaultHttpPropertyExtractor; pub use request_properties::HttpConnectionProperties; pub use request_properties::HttpListenProperties; pub use request_properties::HttpPropertyExtractor; pub use request_properties::HttpRequestProperties; +pub use service::UpgradeUnavailableError; +pub use websocket_upgrade::WebSocketUpgradeError; + +#[derive(Debug, Default, Clone, Copy)] +pub struct Options { + /// By passing a hook function, the caller can customize various configuration + /// options for the HTTP/2 server. + /// See [`http2::Builder`] for what parameters can be customized. + /// + /// If `None`, the default configuration provided by hyper will be used. Note + /// that the default configuration is subject to change in future versions. + pub http2_builder_hook: + Option) -> http2::Builder>, + /// By passing a hook function, the caller can customize various configuration + /// options for the HTTP/1 server. + /// See [`http1::Builder`] for what parameters can be customized. + /// + /// If `None`, the default configuration provided by hyper will be used. Note + /// that the default configuration is subject to change in future versions. + pub http1_builder_hook: Option http1::Builder>, +} deno_core::extension!( deno_http, @@ -111,7 +133,9 @@ deno_core::extension!( http_next::op_http_close_after_finish, http_next::op_http_get_request_header, http_next::op_http_get_request_headers, + http_next::op_http_request_on_cancel, http_next::op_http_get_request_method_and_url, + http_next::op_http_get_request_cancelled, http_next::op_http_read_request_body, http_next::op_http_serve_on, http_next::op_http_serve, @@ -132,8 +156,46 @@ deno_core::extension!( http_next::op_http_cancel, ], esm = ["00_serve.ts", "01_http.js", "02_websocket.ts"], + options = { + options: Options, + }, + state = |state, options| { + state.put::(options.options); + } ); +#[derive(Debug, thiserror::Error)] +pub enum HttpError { + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error(transparent)] + Canceled(#[from] deno_core::Canceled), + #[error("{0}")] + HyperV014(#[source] Arc), + #[error("{0}")] + InvalidHeaderName(#[from] hyper_v014::header::InvalidHeaderName), + #[error("{0}")] + InvalidHeaderValue(#[from] hyper_v014::header::InvalidHeaderValue), + #[error("{0}")] + Http(#[from] hyper_v014::http::Error), + #[error("response headers already sent")] + ResponseHeadersAlreadySent, + #[error("connection closed while sending response")] + ConnectionClosedWhileSendingResponse, + #[error("already in use")] + AlreadyInUse, + #[error("{0}")] + Io(#[from] std::io::Error), + #[error("no response headers")] + NoResponseHeaders, + #[error("response already completed")] + ResponseAlreadyCompleted, + #[error("cannot upgrade because request body was used")] + UpgradeBodyUsed, + #[error(transparent)] + Other(deno_core::error::AnyError), +} + pub enum HttpSocketAddr { IpSocket(std::net::SocketAddr), #[cfg(unix)] @@ -216,7 +278,7 @@ impl HttpConnResource { String, String, )>, - AnyError, + HttpError, > { let fut = async { let (request_tx, request_rx) = oneshot::channel(); @@ -259,8 +321,8 @@ impl HttpConnResource { } /// A future that completes when this HTTP connection is closed or errors. - async fn closed(&self) -> Result<(), AnyError> { - self.closed_fut.clone().map_err(AnyError::from).await + async fn closed(&self) -> Result<(), HttpError> { + self.closed_fut.clone().map_err(HttpError::HyperV014).await } } @@ -280,14 +342,13 @@ pub fn http_create_conn_resource( io: S, addr: A, scheme: &'static str, -) -> Result +) -> ResourceId where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, A: Into, { let conn = HttpConnResource::new(io, scheme, addr.into()); - let rid = state.resource_table.add(conn); - Ok(rid) + state.resource_table.add(conn) } /// An object that implements the `hyper::Service` trait, through which Hyper @@ -423,7 +484,9 @@ impl Resource for HttpStreamReadResource { // safely call `await` on it without creating a race condition. Some(_) => match body.as_mut().next().await.unwrap() { Ok(chunk) => assert!(chunk.is_empty()), - Err(err) => break Err(AnyError::from(err)), + Err(err) => { + break Err(HttpError::HyperV014(Arc::new(err)).into()) + } }, None => break Ok(BufView::empty()), } @@ -545,8 +608,12 @@ struct NextRequestResponse( async fn op_http_accept( state: Rc>, #[smi] rid: ResourceId, -) -> Result, AnyError> { - let conn = state.borrow().resource_table.get::(rid)?; +) -> Result, HttpError> { + let conn = state + .borrow() + .resource_table + .get::(rid) + .map_err(HttpError::Resource)?; match conn.accept().await { Ok(Some((read_stream, write_stream, method, url))) => { @@ -657,11 +724,12 @@ async fn op_http_write_headers( #[smi] status: u16, #[serde] headers: Vec<(ByteString, ByteString)>, #[serde] data: Option, -) -> Result<(), AnyError> { +) -> Result<(), HttpError> { let stream = state .borrow_mut() .resource_table - .get::(rid)?; + .get::(rid) + .map_err(HttpError::Resource)?; // Track supported encoding let encoding = stream.accept_encoding; @@ -708,14 +776,14 @@ async fn op_http_write_headers( let mut old_wr = RcRef::map(&stream, |r| &r.wr).borrow_mut().await; let response_tx = match replace(&mut *old_wr, new_wr) { HttpResponseWriter::Headers(response_tx) => response_tx, - _ => return Err(http_error("response headers already sent")), + _ => return Err(HttpError::ResponseHeadersAlreadySent), }; match response_tx.send(body) { Ok(_) => Ok(()), Err(_) => { stream.conn.closed().await?; - Err(http_error("connection closed while sending response")) + Err(HttpError::ConnectionClosedWhileSendingResponse) } } } @@ -725,11 +793,14 @@ async fn op_http_write_headers( fn op_http_headers( state: &mut OpState, #[smi] rid: u32, -) -> Result, AnyError> { - let stream = state.resource_table.get::(rid)?; +) -> Result, HttpError> { + let stream = state + .resource_table + .get::(rid) + .map_err(HttpError::Resource)?; let rd = RcRef::map(&stream, |r| &r.rd) .try_borrow() - .ok_or_else(|| http_error("already in use"))?; + .ok_or(HttpError::AlreadyInUse)?; match &*rd { HttpRequestReader::Headers(request) => Ok(req_headers(request.headers())), HttpRequestReader::Body(headers, _) => Ok(req_headers(headers)), @@ -741,7 +812,7 @@ fn http_response( data: Option, compressing: bool, encoding: Encoding, -) -> Result<(HttpResponseWriter, hyper_v014::Body), AnyError> { +) -> Result<(HttpResponseWriter, hyper_v014::Body), HttpError> { // Gzip, after level 1, doesn't produce significant size difference. // This default matches nginx default gzip compression level (1): // https://nginx.org/en/docs/http/ngx_http_gzip_module.html#gzip_comp_level @@ -878,25 +949,34 @@ async fn op_http_write_resource( state: Rc>, #[smi] rid: ResourceId, #[smi] stream: ResourceId, -) -> Result<(), AnyError> { +) -> Result<(), HttpError> { let http_stream = state .borrow() .resource_table - .get::(rid)?; + .get::(rid) + .map_err(HttpError::Resource)?; let mut wr = RcRef::map(&http_stream, |r| &r.wr).borrow_mut().await; - let resource = state.borrow().resource_table.get_any(stream)?; + let resource = state + .borrow() + .resource_table + .get_any(stream) + .map_err(HttpError::Resource)?; loop { match *wr { HttpResponseWriter::Headers(_) => { - return Err(http_error("no response headers")) + return Err(HttpError::NoResponseHeaders) } HttpResponseWriter::Closed => { - return Err(http_error("response already completed")) + return Err(HttpError::ResponseAlreadyCompleted) } _ => {} }; - let view = resource.clone().read(64 * 1024).await?; // 64KB + let view = resource + .clone() + .read(64 * 1024) + .await + .map_err(HttpError::Other)?; // 64KB if view.is_empty() { break; } @@ -937,16 +1017,17 @@ async fn op_http_write( state: Rc>, #[smi] rid: ResourceId, #[buffer] buf: JsBuffer, -) -> Result<(), AnyError> { +) -> Result<(), HttpError> { let stream = state .borrow() .resource_table - .get::(rid)?; + .get::(rid) + .map_err(HttpError::Resource)?; let mut wr = RcRef::map(&stream, |r| &r.wr).borrow_mut().await; match &mut *wr { - HttpResponseWriter::Headers(_) => Err(http_error("no response headers")), - HttpResponseWriter::Closed => Err(http_error("response already completed")), + HttpResponseWriter::Headers(_) => Err(HttpError::NoResponseHeaders), + HttpResponseWriter::Closed => Err(HttpError::ResponseAlreadyCompleted), HttpResponseWriter::Body { writer, .. } => { let mut result = writer.write_all(&buf).await; if result.is_ok() { @@ -961,7 +1042,7 @@ async fn op_http_write( stream.conn.closed().await?; // If there was no connection error, drop body_tx. *wr = HttpResponseWriter::Closed; - Err(http_error("response already completed")) + Err(HttpError::ResponseAlreadyCompleted) } } } @@ -975,7 +1056,7 @@ async fn op_http_write( stream.conn.closed().await?; // If there was no connection error, drop body_tx. *wr = HttpResponseWriter::Closed; - Err(http_error("response already completed")) + Err(HttpError::ResponseAlreadyCompleted) } } } @@ -989,11 +1070,12 @@ async fn op_http_write( async fn op_http_shutdown( state: Rc>, #[smi] rid: ResourceId, -) -> Result<(), AnyError> { +) -> Result<(), HttpError> { let stream = state .borrow() .resource_table - .get::(rid)?; + .get::(rid) + .map_err(HttpError::Resource)?; let mut wr = RcRef::map(&stream, |r| &r.wr).borrow_mut().await; let wr = take(&mut *wr); match wr { @@ -1022,14 +1104,12 @@ async fn op_http_shutdown( #[op2] #[string] -fn op_http_websocket_accept_header( - #[string] key: String, -) -> Result { +fn op_http_websocket_accept_header(#[string] key: String) -> String { let digest = ring::digest::digest( &ring::digest::SHA1_FOR_LEGACY_USE_ONLY, format!("{key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11").as_bytes(), ); - Ok(BASE64_STANDARD.encode(digest)) + BASE64_STANDARD.encode(digest) } #[op2(async)] @@ -1037,30 +1117,34 @@ fn op_http_websocket_accept_header( async fn op_http_upgrade_websocket( state: Rc>, #[smi] rid: ResourceId, -) -> Result { +) -> Result { let stream = state .borrow_mut() .resource_table - .get::(rid)?; + .get::(rid) + .map_err(HttpError::Resource)?; let mut rd = RcRef::map(&stream, |r| &r.rd).borrow_mut().await; let request = match &mut *rd { HttpRequestReader::Headers(request) => request, - _ => { - return Err(http_error("cannot upgrade because request body was used")) - } + _ => return Err(HttpError::UpgradeBodyUsed), }; - let (transport, bytes) = - extract_network_stream(hyper_v014::upgrade::on(request).await?); - let ws_rid = - ws_create_server_stream(&mut state.borrow_mut(), transport, bytes)?; - Ok(ws_rid) + let (transport, bytes) = extract_network_stream( + hyper_v014::upgrade::on(request) + .await + .map_err(|err| HttpError::HyperV014(Arc::new(err)))?, + ); + Ok(ws_create_server_stream( + &mut state.borrow_mut(), + transport, + bytes, + )) } // Needed so hyper can use non Send futures #[derive(Clone)] -struct LocalExecutor; +pub struct LocalExecutor; impl hyper_v014::rt::Executor for LocalExecutor where @@ -1082,10 +1166,6 @@ where } } -fn http_error(message: &'static str) -> AnyError { - custom_error("Http", message) -} - /// Filters out the ever-surprising 'shutdown ENOTCONN' errors. fn filter_enotconn( result: Result<(), hyper_v014::Error>, diff --git a/ext/http/request_body.rs b/ext/http/request_body.rs index 45df12457cc278..f1c3f358ea977b 100644 --- a/ext/http/request_body.rs +++ b/ext/http/request_body.rs @@ -1,9 +1,9 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use bytes::Bytes; -use deno_core::error::AnyError; use deno_core::futures::stream::Peekable; use deno_core::futures::Stream; use deno_core::futures::StreamExt; +use deno_core::futures::TryFutureExt; use deno_core::AsyncRefCell; use deno_core::AsyncResult; use deno_core::BufView; @@ -22,7 +22,7 @@ use std::task::Poll; struct ReadFuture(Incoming); impl Stream for ReadFuture { - type Item = Result; + type Item = Result; fn poll_next( self: Pin<&mut Self>, @@ -37,13 +37,13 @@ impl Stream for ReadFuture { if let Ok(data) = frame.into_data() { // Ensure that we never yield an empty frame if !data.is_empty() { - break Poll::Ready(Some(Ok::<_, AnyError>(data))); + break Poll::Ready(Some(Ok(data))); } } // Loop again so we don't lose the waker continue; } - Some(Err(e)) => Poll::Ready(Some(Err(e.into()))), + Some(Err(e)) => Poll::Ready(Some(Err(e))), None => Poll::Ready(None), }; } @@ -58,7 +58,7 @@ impl HttpRequestBody { Self(AsyncRefCell::new(ReadFuture(body).peekable()), size_hint) } - async fn read(self: Rc, limit: usize) -> Result { + async fn read(self: Rc, limit: usize) -> Result { let peekable = RcRef::map(self, |this| &this.0); let mut peekable = peekable.borrow_mut().await; match Pin::new(&mut *peekable).peek_mut().await { @@ -82,7 +82,7 @@ impl Resource for HttpRequestBody { } fn read(self: Rc, limit: usize) -> AsyncResult { - Box::pin(HttpRequestBody::read(self, limit)) + Box::pin(HttpRequestBody::read(self, limit).map_err(Into::into)) } fn size_hint(&self) -> (u64, Option) { diff --git a/ext/http/request_properties.rs b/ext/http/request_properties.rs index 1422c7417d2774..39d35a79f14b82 100644 --- a/ext/http/request_properties.rs +++ b/ext/http/request_properties.rs @@ -34,8 +34,8 @@ pub struct HttpConnectionProperties { pub stream_type: NetworkStreamType, } -pub struct HttpRequestProperties { - pub authority: Option, +pub struct HttpRequestProperties<'a> { + pub authority: Option>, } /// Pluggable trait to determine listen, connection and request properties @@ -84,11 +84,11 @@ pub trait HttpPropertyExtractor { ) -> NetworkStream; /// Determines the request properties. - fn request_properties( - connection_properties: &HttpConnectionProperties, - uri: &Uri, - headers: &HeaderMap, - ) -> HttpRequestProperties; + fn request_properties<'a>( + connection_properties: &'a HttpConnectionProperties, + uri: &'a Uri, + headers: &'a HeaderMap, + ) -> HttpRequestProperties<'a>; } pub struct DefaultHttpPropertyExtractor {} @@ -180,18 +180,17 @@ impl HttpPropertyExtractor for DefaultHttpPropertyExtractor { } } - fn request_properties( - connection_properties: &HttpConnectionProperties, - uri: &Uri, - headers: &HeaderMap, - ) -> HttpRequestProperties { + fn request_properties<'a>( + connection_properties: &'a HttpConnectionProperties, + uri: &'a Uri, + headers: &'a HeaderMap, + ) -> HttpRequestProperties<'a> { let authority = req_host( uri, headers, connection_properties.stream_type, connection_properties.local_port.unwrap_or_default(), - ) - .map(|s| s.into_owned()); + ); HttpRequestProperties { authority } } diff --git a/ext/http/service.rs b/ext/http/service.rs index 787e9babf848d9..ce24dea43f372c 100644 --- a/ext/http/service.rs +++ b/ext/http/service.rs @@ -2,7 +2,6 @@ use crate::request_properties::HttpConnectionProperties; use crate::response_body::ResponseBytesInner; use crate::response_body::ResponseStreamResult; -use deno_core::error::AnyError; use deno_core::futures::ready; use deno_core::BufView; use deno_core::OpState; @@ -28,6 +27,7 @@ use std::rc::Rc; use std::task::Context; use std::task::Poll; use std::task::Waker; +use tokio::sync::oneshot; pub type Request = hyper::Request; pub type Response = hyper::Response; @@ -206,8 +206,13 @@ pub(crate) async fn handle_request( Ok(response) } +#[derive(Debug, thiserror::Error)] +#[error("upgrade unavailable")] +pub struct UpgradeUnavailableError; + struct HttpRecordInner { server_state: SignallingRc, + closed_channel: Option>, request_info: HttpConnectionProperties, request_parts: http::request::Parts, request_body: Option, @@ -273,6 +278,7 @@ impl HttpRecord { response_body_finished: false, response_body_waker: None, trailers: None, + closed_channel: None, been_dropped: false, finished: false, needs_close_after_finish: false, @@ -309,6 +315,10 @@ impl HttpRecord { RefMut::map(self.self_mut(), |inner| &mut inner.needs_close_after_finish) } + pub fn on_cancel(&self, sender: oneshot::Sender<()>) { + self.self_mut().closed_channel = Some(sender); + } + fn recycle(self: Rc) { assert!( Rc::strong_count(&self) == 1, @@ -344,14 +354,14 @@ impl HttpRecord { } /// Perform the Hyper upgrade on this record. - pub fn upgrade(&self) -> Result { + pub fn upgrade(&self) -> Result { // Manually perform the upgrade. We're peeking into hyper's underlying machinery here a bit self .self_mut() .request_parts .extensions .remove::() - .ok_or_else(|| AnyError::msg("upgrade unavailable")) + .ok_or(UpgradeUnavailableError) } /// Take the Hyper body from this record. @@ -387,6 +397,9 @@ impl HttpRecord { inner.been_dropped = true; // The request body might include actual resources. inner.request_body.take(); + if let Some(closed_channel) = inner.closed_channel.take() { + let _ = closed_channel.send(()); + } } /// Complete this record, potentially expunging it if it is fully complete (ie: cancelled as well). @@ -515,7 +528,7 @@ pub struct HttpRecordResponse(ManuallyDrop>); impl Body for HttpRecordResponse { type Data = BufView; - type Error = AnyError; + type Error = deno_core::error::AnyError; fn poll_frame( self: Pin<&mut Self>, @@ -640,7 +653,7 @@ mod tests { } #[tokio::test] - async fn test_handle_request() -> Result<(), AnyError> { + async fn test_handle_request() -> Result<(), deno_core::error::AnyError> { let (tx, mut rx) = tokio::sync::mpsc::channel(10); let server_state = HttpServerState::new(); let server_state_check = server_state.clone(); diff --git a/ext/http/websocket_upgrade.rs b/ext/http/websocket_upgrade.rs index 4dead767a3e266..af9504717e3948 100644 --- a/ext/http/websocket_upgrade.rs +++ b/ext/http/websocket_upgrade.rs @@ -4,7 +4,6 @@ use std::marker::PhantomData; use bytes::Bytes; use bytes::BytesMut; -use deno_core::error::AnyError; use httparse::Status; use hyper::header::HeaderName; use hyper::header::HeaderValue; @@ -13,12 +12,30 @@ use memmem::Searcher; use memmem::TwoWaySearcher; use once_cell::sync::OnceCell; -use crate::http_error; +#[derive(Debug, thiserror::Error)] +pub enum WebSocketUpgradeError { + #[error("invalid headers")] + InvalidHeaders, + #[error("{0}")] + HttpParse(#[from] httparse::Error), + #[error("{0}")] + Http(#[from] http::Error), + #[error("{0}")] + Utf8(#[from] std::str::Utf8Error), + #[error("{0}")] + InvalidHeaderName(#[from] http::header::InvalidHeaderName), + #[error("{0}")] + InvalidHeaderValue(#[from] http::header::InvalidHeaderValue), + #[error("invalid HTTP status line")] + InvalidHttpStatusLine, + #[error("attempted to write to completed upgrade buffer")] + UpgradeBufferAlreadyCompleted, +} /// Given a buffer that ends in `\n\n` or `\r\n\r\n`, returns a parsed [`Request`]. fn parse_response( header_bytes: &[u8], -) -> Result<(usize, Response), AnyError> { +) -> Result<(usize, Response), WebSocketUpgradeError> { let mut headers = [httparse::EMPTY_HEADER; 16]; let status = httparse::parse_headers(header_bytes, &mut headers)?; match status { @@ -32,7 +49,7 @@ fn parse_response( } Ok((index, resp)) } - _ => Err(http_error("invalid headers")), + _ => Err(WebSocketUpgradeError::InvalidHeaders), } } @@ -69,11 +86,14 @@ pub struct WebSocketUpgrade { impl WebSocketUpgrade { /// Ensures that the status line starts with "HTTP/1.1 101 " which matches all of the node.js /// WebSocket libraries that are known. We don't care about the trailing status text. - fn validate_status(&self, status: &[u8]) -> Result<(), AnyError> { + fn validate_status( + &self, + status: &[u8], + ) -> Result<(), WebSocketUpgradeError> { if status.starts_with(b"HTTP/1.1 101 ") { Ok(()) } else { - Err(http_error("invalid HTTP status line")) + Err(WebSocketUpgradeError::InvalidHttpStatusLine) } } @@ -82,7 +102,7 @@ impl WebSocketUpgrade { pub fn write( &mut self, bytes: &[u8], - ) -> Result, Bytes)>, AnyError> { + ) -> Result, Bytes)>, WebSocketUpgradeError> { use WebSocketUpgradeState::*; match self.state { @@ -142,9 +162,7 @@ impl WebSocketUpgrade { Ok(None) } } - Complete => { - Err(http_error("attempted to write to completed upgrade buffer")) - } + Complete => Err(WebSocketUpgradeError::UpgradeBufferAlreadyCompleted), } } } @@ -157,8 +175,8 @@ mod tests { type ExpectedResponseAndHead = Option<(Response, &'static [u8])>; fn assert_response( - result: Result, Bytes)>, AnyError>, - expected: Result, + result: Result, Bytes)>, WebSocketUpgradeError>, + expected: Result, chunk_info: Option<(usize, usize)>, ) { let formatted = format!("{result:?}"); @@ -189,8 +207,8 @@ mod tests { "Expected Ok(None), was {formatted}", ), Err(e) => assert_eq!( - e, - result.err().map(|e| format!("{e:?}")).unwrap_or_default(), + format!("{e:?}"), + format!("{:?}", result.unwrap_err()), "Expected error, was {formatted}", ), } @@ -198,7 +216,7 @@ mod tests { fn validate_upgrade_all_at_once( s: &str, - expected: Result, + expected: Result, ) { let mut upgrade = WebSocketUpgrade::default(); let res = upgrade.write(s.as_bytes()); @@ -209,7 +227,7 @@ mod tests { fn validate_upgrade_chunks( s: &str, size: usize, - expected: Result, + expected: Result, ) { let chunk_info = Some((s.as_bytes().len(), size)); let mut upgrade = WebSocketUpgrade::default(); @@ -226,7 +244,7 @@ mod tests { fn validate_upgrade( s: &str, - expected: fn() -> Result, + expected: fn() -> Result, ) { validate_upgrade_all_at_once(s, expected()); validate_upgrade_chunks(s, 1, expected()); @@ -315,7 +333,7 @@ mod tests { #[test] fn upgrade_invalid_status() { validate_upgrade("HTTP/1.1 200 OK\nConnection: Upgrade\n\n", || { - Err("invalid HTTP status line") + Err(WebSocketUpgradeError::InvalidHttpStatusLine) }); } @@ -327,7 +345,11 @@ mod tests { .join("\n"); validate_upgrade( &format!("HTTP/1.1 101 Switching Protocols\n{headers}\n\n"), - || Err("too many headers"), + || { + Err(WebSocketUpgradeError::HttpParse( + httparse::Error::TooManyHeaders, + )) + }, ); } } diff --git a/ext/io/Cargo.toml b/ext/io/Cargo.toml index 22c346764bed85..b8a14acd6f1f9d 100644 --- a/ext/io/Cargo.toml +++ b/ext/io/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_io" -version = "0.78.0" +version = "0.90.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/io/bi_pipe.rs b/ext/io/bi_pipe.rs index 04fff7b00b44a1..3492e2f441e823 100644 --- a/ext/io/bi_pipe.rs +++ b/ext/io/bi_pipe.rs @@ -2,7 +2,6 @@ use std::rc::Rc; -use deno_core::error::AnyError; use deno_core::AsyncRefCell; use deno_core::AsyncResult; use deno_core::CancelHandle; @@ -11,11 +10,7 @@ use deno_core::RcRef; use tokio::io::AsyncReadExt; use tokio::io::AsyncWriteExt; -#[cfg(unix)] -pub type RawBiPipeHandle = std::os::fd::RawFd; - -#[cfg(windows)] -pub type RawBiPipeHandle = std::os::windows::io::RawHandle; +pub type RawBiPipeHandle = super::RawIoHandle; /// One end of a bidirectional pipe. This implements the /// `Resource` trait. @@ -75,13 +70,16 @@ impl BiPipeResource { pub async fn read( self: Rc, data: &mut [u8], - ) -> Result { + ) -> Result { let mut rd = RcRef::map(&self, |r| &r.read_half).borrow_mut().await; let cancel_handle = RcRef::map(&self, |r| &r.cancel); - Ok(rd.read(data).try_or_cancel(cancel_handle).await?) + rd.read(data).try_or_cancel(cancel_handle).await } - pub async fn write(self: Rc, data: &[u8]) -> Result { + pub async fn write( + self: Rc, + data: &[u8], + ) -> Result { let mut wr = RcRef::map(self, |r| &r.write_half).borrow_mut().await; let nwritten = wr.write(data).await?; wr.flush().await?; @@ -185,9 +183,10 @@ fn from_raw( ) -> Result<(BiPipeRead, BiPipeWrite), std::io::Error> { use std::os::fd::FromRawFd; // Safety: The fd is part of a pair of connected sockets - let unix_stream = tokio::net::UnixStream::from_std(unsafe { - std::os::unix::net::UnixStream::from_raw_fd(stream) - })?; + let unix_stream = + unsafe { std::os::unix::net::UnixStream::from_raw_fd(stream) }; + unix_stream.set_nonblocking(true)?; + let unix_stream = tokio::net::UnixStream::from_std(unix_stream)?; let (read, write) = unix_stream.into_split(); Ok((BiPipeRead { inner: read }, BiPipeWrite { inner: write })) } @@ -274,15 +273,15 @@ impl_async_write!(for BiPipe -> self.write_end); /// Creates both sides of a bidirectional pipe, returning the raw /// handles to the underlying OS resources. -pub fn bi_pipe_pair_raw() -> Result<(RawBiPipeHandle, RawBiPipeHandle), AnyError> -{ +pub fn bi_pipe_pair_raw( +) -> Result<(RawBiPipeHandle, RawBiPipeHandle), std::io::Error> { #[cfg(unix)] { // SockFlag is broken on macOS // https://github.com/nix-rust/nix/issues/861 let mut fds = [-1, -1]; #[cfg(not(target_os = "macos"))] - let flags = libc::SOCK_CLOEXEC | libc::SOCK_NONBLOCK; + let flags = libc::SOCK_CLOEXEC; #[cfg(target_os = "macos")] let flags = 0; @@ -297,19 +296,19 @@ pub fn bi_pipe_pair_raw() -> Result<(RawBiPipeHandle, RawBiPipeHandle), AnyError ) }; if ret != 0 { - return Err(std::io::Error::last_os_error().into()); + return Err(std::io::Error::last_os_error()); } if cfg!(target_os = "macos") { let fcntl = |fd: i32, flag: libc::c_int| -> Result<(), std::io::Error> { // SAFETY: libc call, fd is valid - let flags = unsafe { libc::fcntl(fd, libc::F_GETFL) }; + let flags = unsafe { libc::fcntl(fd, libc::F_GETFD) }; if flags == -1 { return Err(fail(fds)); } // SAFETY: libc call, fd is valid - let ret = unsafe { libc::fcntl(fd, libc::F_SETFL, flags | flag) }; + let ret = unsafe { libc::fcntl(fd, libc::F_SETFD, flags | flag) }; if ret == -1 { return Err(fail(fds)); } @@ -325,13 +324,9 @@ pub fn bi_pipe_pair_raw() -> Result<(RawBiPipeHandle, RawBiPipeHandle), AnyError std::io::Error::last_os_error() } - // SOCK_NONBLOCK is not supported on macOS. - (fcntl)(fds[0], libc::O_NONBLOCK)?; - (fcntl)(fds[1], libc::O_NONBLOCK)?; - // SOCK_CLOEXEC is not supported on macOS. - (fcntl)(fds[0], libc::FD_CLOEXEC)?; - (fcntl)(fds[1], libc::FD_CLOEXEC)?; + fcntl(fds[0], libc::FD_CLOEXEC)?; + fcntl(fds[1], libc::FD_CLOEXEC)?; } let fd1 = fds[0]; @@ -393,7 +388,7 @@ pub fn bi_pipe_pair_raw() -> Result<(RawBiPipeHandle, RawBiPipeHandle), AnyError continue; } - return Err(err.into()); + return Err(err); } break (path, hd1); @@ -415,7 +410,7 @@ pub fn bi_pipe_pair_raw() -> Result<(RawBiPipeHandle, RawBiPipeHandle), AnyError 0, ); if hd2 == INVALID_HANDLE_VALUE { - return Err(io::Error::last_os_error().into()); + return Err(io::Error::last_os_error()); } // Will not block because we have create the pair. @@ -423,7 +418,7 @@ pub fn bi_pipe_pair_raw() -> Result<(RawBiPipeHandle, RawBiPipeHandle), AnyError let err = std::io::Error::last_os_error(); if err.raw_os_error() != Some(ERROR_PIPE_CONNECTED as i32) { CloseHandle(hd2); - return Err(err.into()); + return Err(err); } } diff --git a/ext/io/fs.rs b/ext/io/fs.rs index 3798c1429da0c5..bd5dfd0bb98a64 100644 --- a/ext/io/fs.rs +++ b/ext/io/fs.rs @@ -1,15 +1,12 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use std::borrow::Cow; +use std::fmt::Formatter; use std::io; use std::rc::Rc; use std::time::SystemTime; use std::time::UNIX_EPOCH; -use deno_core::error::custom_error; -use deno_core::error::not_supported; -use deno_core::error::resource_unavailable; -use deno_core::error::AnyError; use deno_core::BufMutView; use deno_core::BufView; use deno_core::OpState; @@ -25,6 +22,21 @@ pub enum FsError { NotCapable(&'static str), } +impl std::fmt::Display for FsError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + FsError::Io(err) => std::fmt::Display::fmt(err, f), + FsError::FileBusy => f.write_str("file busy"), + FsError::NotSupported => f.write_str("not supported"), + FsError::NotCapable(err) => { + f.write_str(&format!("requires {err} access")) + } + } + } +} + +impl std::error::Error for FsError {} + impl FsError { pub fn kind(&self) -> io::ErrorKind { match self { @@ -59,19 +71,6 @@ impl From for FsError { } } -impl From for AnyError { - fn from(err: FsError) -> Self { - match err { - FsError::Io(err) => AnyError::from(err), - FsError::FileBusy => resource_unavailable(), - FsError::NotSupported => not_supported(), - FsError::NotCapable(err) => { - custom_error("NotCapable", format!("permission denied: {err}")) - } - } - } -} - impl From for FsError { fn from(err: JoinError) -> Self { if err.is_cancelled() { @@ -95,6 +94,7 @@ pub struct FsStat { pub mtime: Option, pub atime: Option, pub birthtime: Option, + pub ctime: Option, pub dev: u64, pub ino: u64, @@ -154,6 +154,16 @@ impl FsStat { } } + #[inline(always)] + fn get_ctime(ctime_or_0: i64) -> Option { + if ctime_or_0 > 0 { + // ctime return seconds since epoch, but we need milliseconds + return Some(ctime_or_0 as u64 * 1000); + } + + None + } + Self { is_file: metadata.is_file(), is_directory: metadata.is_dir(), @@ -163,6 +173,7 @@ impl FsStat { mtime: to_msec(metadata.modified()), atime: to_msec(metadata.accessed()), birthtime: to_msec(metadata.created()), + ctime: get_ctime(unix_or_zero!(ctime)), dev: unix_or_zero!(dev), ino: unix_or_zero!(ino), @@ -204,8 +215,8 @@ pub trait File { fn write_all_sync(self: Rc, buf: &[u8]) -> FsResult<()>; async fn write_all(self: Rc, buf: BufView) -> FsResult<()>; - fn read_all_sync(self: Rc) -> FsResult>; - async fn read_all_async(self: Rc) -> FsResult>; + fn read_all_sync(self: Rc) -> FsResult>; + async fn read_all_async(self: Rc) -> FsResult>; fn chmod_sync(self: Rc, pathmode: u32) -> FsResult<()>; async fn chmod_async(self: Rc, mode: u32) -> FsResult<()>; @@ -266,9 +277,9 @@ impl FileResource { state: &OpState, rid: ResourceId, f: F, - ) -> Result + ) -> Result where - F: FnOnce(Rc) -> Result, + F: FnOnce(Rc) -> Result, { let resource = state.resource_table.get::(rid)?; f(resource) @@ -277,7 +288,7 @@ impl FileResource { pub fn get_file( state: &OpState, rid: ResourceId, - ) -> Result, AnyError> { + ) -> Result, deno_core::error::AnyError> { let resource = state.resource_table.get::(rid)?; Ok(resource.file()) } @@ -286,9 +297,9 @@ impl FileResource { state: &OpState, rid: ResourceId, f: F, - ) -> Result + ) -> Result where - F: FnOnce(Rc) -> Result, + F: FnOnce(Rc) -> Result, { Self::with_resource(state, rid, |r| f(r.file.clone())) } @@ -303,10 +314,7 @@ impl deno_core::Resource for FileResource { Cow::Borrowed(&self.name) } - fn read( - self: Rc, - limit: usize, - ) -> deno_core::AsyncResult { + fn read(self: Rc, limit: usize) -> deno_core::AsyncResult { Box::pin(async move { self .file @@ -319,8 +327,8 @@ impl deno_core::Resource for FileResource { fn read_byob( self: Rc, - buf: deno_core::BufMutView, - ) -> deno_core::AsyncResult<(usize, deno_core::BufMutView)> { + buf: BufMutView, + ) -> deno_core::AsyncResult<(usize, BufMutView)> { Box::pin(async move { self .file @@ -333,17 +341,14 @@ impl deno_core::Resource for FileResource { fn write( self: Rc, - buf: deno_core::BufView, + buf: BufView, ) -> deno_core::AsyncResult { Box::pin(async move { self.file.clone().write(buf).await.map_err(|err| err.into()) }) } - fn write_all( - self: Rc, - buf: deno_core::BufView, - ) -> deno_core::AsyncResult<()> { + fn write_all(self: Rc, buf: BufView) -> deno_core::AsyncResult<()> { Box::pin(async move { self .file diff --git a/ext/io/lib.rs b/ext/io/lib.rs index 47921bcee41bad..873fccd7b89871 100644 --- a/ext/io/lib.rs +++ b/ext/io/lib.rs @@ -1,6 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::AnyError; use deno_core::op2; use deno_core::unsync::spawn_blocking; use deno_core::unsync::TaskQueue; @@ -48,6 +47,7 @@ use winapi::um::processenv::GetStdHandle; #[cfg(windows)] use winapi::um::winbase; +use deno_core::futures::TryFutureExt; #[cfg(windows)] use parking_lot::Condvar; #[cfg(windows)] @@ -67,6 +67,7 @@ pub use pipe::AsyncPipeRead; pub use pipe::AsyncPipeWrite; pub use pipe::PipeRead; pub use pipe::PipeWrite; +pub use pipe::RawPipeHandle; pub use bi_pipe::bi_pipe_pair_raw; pub use bi_pipe::BiPipe; @@ -75,6 +76,112 @@ pub use bi_pipe::BiPipeResource; pub use bi_pipe::BiPipeWrite; pub use bi_pipe::RawBiPipeHandle; +/// Abstraction over `AsRawFd` (unix) and `AsRawHandle` (windows) +pub trait AsRawIoHandle { + fn as_raw_io_handle(&self) -> RawIoHandle; +} + +#[cfg(unix)] +impl AsRawIoHandle for T +where + T: std::os::unix::io::AsRawFd, +{ + fn as_raw_io_handle(&self) -> RawIoHandle { + self.as_raw_fd() + } +} + +#[cfg(windows)] +impl AsRawIoHandle for T +where + T: std::os::windows::io::AsRawHandle, +{ + fn as_raw_io_handle(&self) -> RawIoHandle { + self.as_raw_handle() + } +} + +/// Abstraction over `IntoRawFd` (unix) and `IntoRawHandle` (windows) +pub trait IntoRawIoHandle { + fn into_raw_io_handle(self) -> RawIoHandle; +} + +#[cfg(unix)] +impl IntoRawIoHandle for T +where + T: std::os::unix::io::IntoRawFd, +{ + fn into_raw_io_handle(self) -> RawIoHandle { + self.into_raw_fd() + } +} + +#[cfg(windows)] +impl IntoRawIoHandle for T +where + T: std::os::windows::io::IntoRawHandle, +{ + fn into_raw_io_handle(self) -> RawIoHandle { + self.into_raw_handle() + } +} + +/// Abstraction over `FromRawFd` (unix) and `FromRawHandle` (windows) +pub trait FromRawIoHandle: Sized { + /// Constructs a type from a raw io handle (fd/HANDLE). + /// + /// # Safety + /// + /// Refer to the standard library docs ([unix](https://doc.rust-lang.org/stable/std/os/windows/io/trait.FromRawHandle.html#tymethod.from_raw_handle)) ([windows](https://doc.rust-lang.org/stable/std/os/fd/trait.FromRawFd.html#tymethod.from_raw_fd)) + /// + unsafe fn from_raw_io_handle(handle: RawIoHandle) -> Self; +} + +#[cfg(unix)] +impl FromRawIoHandle for T +where + T: std::os::unix::io::FromRawFd, +{ + unsafe fn from_raw_io_handle(fd: RawIoHandle) -> T { + // SAFETY: upheld by caller + unsafe { T::from_raw_fd(fd) } + } +} + +#[cfg(windows)] +impl FromRawIoHandle for T +where + T: std::os::windows::io::FromRawHandle, +{ + unsafe fn from_raw_io_handle(fd: RawIoHandle) -> T { + // SAFETY: upheld by caller + unsafe { T::from_raw_handle(fd) } + } +} + +#[cfg(unix)] +pub type RawIoHandle = std::os::fd::RawFd; + +#[cfg(windows)] +pub type RawIoHandle = std::os::windows::io::RawHandle; + +pub fn close_raw_handle(handle: RawIoHandle) { + #[cfg(unix)] + { + // SAFETY: libc call + unsafe { + libc::close(handle); + } + } + #[cfg(windows)] + { + // SAFETY: win32 call + unsafe { + windows_sys::Win32::Foundation::CloseHandle(handle as _); + } + } +} + // Store the stdio fd/handles in global statics in order to keep them // alive for the duration of the application since the last handle/fd // being dropped will close the corresponding pipe. @@ -241,13 +348,13 @@ where RcRef::map(self, |r| &r.stream).borrow_mut() } - async fn write(self: Rc, data: &[u8]) -> Result { + async fn write(self: Rc, data: &[u8]) -> Result { let mut stream = self.borrow_mut().await; let nwritten = stream.write(data).await?; Ok(nwritten) } - async fn shutdown(self: Rc) -> Result<(), AnyError> { + async fn shutdown(self: Rc) -> Result<(), io::Error> { let mut stream = self.borrow_mut().await; stream.shutdown().await?; Ok(()) @@ -289,7 +396,7 @@ where self.cancel_handle.cancel() } - async fn read(self: Rc, data: &mut [u8]) -> Result { + async fn read(self: Rc, data: &mut [u8]) -> Result { let mut rd = self.borrow_mut().await; let nread = rd.read(data).try_or_cancel(self.cancel_handle()).await?; Ok(nread) @@ -310,7 +417,7 @@ impl Resource for ChildStdinResource { deno_core::impl_writable!(); fn shutdown(self: Rc) -> AsyncResult<()> { - Box::pin(self.shutdown()) + Box::pin(self.shutdown().map_err(|e| e.into())) } } @@ -682,26 +789,26 @@ impl crate::fs::File for StdFileResourceInner { } } - fn read_all_sync(self: Rc) -> FsResult> { + fn read_all_sync(self: Rc) -> FsResult> { match self.kind { StdFileResourceKind::File | StdFileResourceKind::Stdin(_) => { let mut buf = Vec::new(); self.with_sync(|file| Ok(file.read_to_end(&mut buf)?))?; - Ok(buf) + Ok(Cow::Owned(buf)) } StdFileResourceKind::Stdout | StdFileResourceKind::Stderr => { Err(FsError::NotSupported) } } } - async fn read_all_async(self: Rc) -> FsResult> { + async fn read_all_async(self: Rc) -> FsResult> { match self.kind { StdFileResourceKind::File | StdFileResourceKind::Stdin(_) => { self .with_inner_blocking_task(|file| { let mut buf = Vec::new(); file.read_to_end(&mut buf)?; - Ok(buf) + Ok(Cow::Owned(buf)) }) .await } @@ -903,7 +1010,7 @@ pub fn op_print( state: &mut OpState, #[string] msg: &str, is_err: bool, -) -> Result<(), AnyError> { +) -> Result<(), deno_core::error::AnyError> { let rid = if is_err { 2 } else { 1 }; FileResource::with_file(state, rid, move |file| { Ok(file.write_all_sync(msg.as_bytes())?) diff --git a/ext/io/pipe.rs b/ext/io/pipe.rs index 70788f7520974b..e0e019e277dcb6 100644 --- a/ext/io/pipe.rs +++ b/ext/io/pipe.rs @@ -3,6 +3,8 @@ use std::io; use std::pin::Pin; use std::process::Stdio; +pub type RawPipeHandle = super::RawIoHandle; + // The synchronous read end of a unidirectional pipe. pub struct PipeRead { file: std::fs::File, diff --git a/ext/kv/Cargo.toml b/ext/kv/Cargo.toml index eee4762eb42cfb..3d050d9bbb1232 100644 --- a/ext/kv/Cargo.toml +++ b/ext/kv/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_kv" -version = "0.76.0" +version = "0.88.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -17,10 +17,12 @@ path = "lib.rs" anyhow.workspace = true async-trait.workspace = true base64.workspace = true +boxed_error.workspace = true bytes.workspace = true chrono = { workspace = true, features = ["now"] } deno_core.workspace = true deno_fetch.workspace = true +deno_path_util.workspace = true deno_permissions.workspace = true deno_tls.workspace = true denokv_proto.workspace = true @@ -35,6 +37,7 @@ prost.workspace = true rand.workspace = true rusqlite.workspace = true serde.workspace = true +thiserror.workspace = true url.workspace = true [build-dependencies] diff --git a/ext/kv/config.rs b/ext/kv/config.rs index 6e2e2c3a1f8b1a..7166bcbcc282a6 100644 --- a/ext/kv/config.rs +++ b/ext/kv/config.rs @@ -1,16 +1,17 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +#[derive(Clone, Copy, Debug)] pub struct KvConfig { - pub(crate) max_write_key_size_bytes: usize, - pub(crate) max_read_key_size_bytes: usize, - pub(crate) max_value_size_bytes: usize, - pub(crate) max_read_ranges: usize, - pub(crate) max_read_entries: usize, - pub(crate) max_checks: usize, - pub(crate) max_mutations: usize, - pub(crate) max_watched_keys: usize, - pub(crate) max_total_mutation_size_bytes: usize, - pub(crate) max_total_key_size_bytes: usize, + pub max_write_key_size_bytes: usize, + pub max_read_key_size_bytes: usize, + pub max_value_size_bytes: usize, + pub max_read_ranges: usize, + pub max_read_entries: usize, + pub max_checks: usize, + pub max_mutations: usize, + pub max_watched_keys: usize, + pub max_total_mutation_size_bytes: usize, + pub max_total_key_size_bytes: usize, } impl KvConfig { diff --git a/ext/kv/lib.rs b/ext/kv/lib.rs index 13e4f1662fce6c..ce7509721ae0e2 100644 --- a/ext/kv/lib.rs +++ b/ext/kv/lib.rs @@ -12,15 +12,12 @@ use std::num::NonZeroU32; use std::rc::Rc; use std::time::Duration; -use anyhow::bail; use base64::prelude::BASE64_URL_SAFE; use base64::Engine; +use boxed_error::Boxed; use chrono::DateTime; use chrono::Utc; -use deno_core::anyhow::Context; use deno_core::error::get_custom_error_class; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::futures::StreamExt; use deno_core::op2; use deno_core::serde_v8::AnyValue; @@ -118,12 +115,75 @@ impl Resource for DatabaseWatcherResource { } } -#[op2(async)] +#[derive(Debug, Boxed)] +pub struct KvError(pub Box); + +#[derive(Debug, thiserror::Error)] +pub enum KvErrorKind { + #[error(transparent)] + DatabaseHandler(deno_core::error::AnyError), + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error("Too many ranges (max {0})")] + TooManyRanges(usize), + #[error("Too many entries (max {0})")] + TooManyEntries(usize), + #[error("Too many checks (max {0})")] + TooManyChecks(usize), + #[error("Too many mutations (max {0})")] + TooManyMutations(usize), + #[error("Too many keys (max {0})")] + TooManyKeys(usize), + #[error("limit must be greater than 0")] + InvalidLimit, + #[error("Invalid boundary key")] + InvalidBoundaryKey, + #[error("Key too large for read (max {0} bytes)")] + KeyTooLargeToRead(usize), + #[error("Key too large for write (max {0} bytes)")] + KeyTooLargeToWrite(usize), + #[error("Total mutation size too large (max {0} bytes)")] + TotalMutationTooLarge(usize), + #[error("Total key size too large (max {0} bytes)")] + TotalKeyTooLarge(usize), + #[error(transparent)] + Kv(deno_core::error::AnyError), + #[error(transparent)] + Io(#[from] std::io::Error), + #[error("Queue message not found")] + QueueMessageNotFound, + #[error("Start key is not in the keyspace defined by prefix")] + StartKeyNotInKeyspace, + #[error("End key is not in the keyspace defined by prefix")] + EndKeyNotInKeyspace, + #[error("Start key is greater than end key")] + StartKeyGreaterThanEndKey, + #[error("Invalid check")] + InvalidCheck(#[source] KvCheckError), + #[error("Invalid mutation")] + InvalidMutation(#[source] KvMutationError), + #[error("Invalid enqueue")] + InvalidEnqueue(#[source] std::io::Error), + #[error("key cannot be empty")] + EmptyKey, // TypeError + #[error("Value too large (max {0} bytes)")] + ValueTooLarge(usize), // TypeError + #[error("enqueue payload too large (max {0} bytes)")] + EnqueuePayloadTooLarge(usize), // TypeError + #[error("invalid cursor")] + InvalidCursor, + #[error("cursor out of bounds")] + CursorOutOfBounds, + #[error("Invalid range")] + InvalidRange, +} + +#[op2(async, stack_trace)] #[smi] async fn op_kv_database_open( state: Rc>, #[string] path: Option, -) -> Result +) -> Result where DBH: DatabaseHandler + 'static, { @@ -134,7 +194,10 @@ where .check_or_exit(UNSTABLE_FEATURE_NAME, "Deno.openKv"); state.borrow::>().clone() }; - let db = handler.open(state.clone(), path).await?; + let db = handler + .open(state.clone(), path) + .await + .map_err(KvErrorKind::DatabaseHandler)?; let rid = state.borrow_mut().resource_table.add(DatabaseResource { db, cancel_handle: CancelHandle::new_rc(), @@ -184,8 +247,8 @@ enum ToV8Value { } impl TryFrom for KvValue { - type Error = AnyError; - fn try_from(value: FromV8Value) -> Result { + type Error = num_bigint::TryFromBigIntError; + fn try_from(value: FromV8Value) -> Result { Ok(match value { FromV8Value::V8(buf) => KvValue::V8(buf.to_vec()), FromV8Value::Bytes(buf) => KvValue::Bytes(buf.to_vec()), @@ -214,8 +277,8 @@ struct ToV8KvEntry { } impl TryFrom for ToV8KvEntry { - type Error = AnyError; - fn try_from(entry: KvEntry) -> Result { + type Error = std::io::Error; + fn try_from(entry: KvEntry) -> Result { Ok(ToV8KvEntry { key: decode_key(&entry.key)? .0 @@ -261,14 +324,16 @@ async fn op_kv_snapshot_read( #[smi] rid: ResourceId, #[serde] ranges: Vec, #[serde] consistency: V8Consistency, -) -> Result>, AnyError> +) -> Result>, KvError> where DBH: DatabaseHandler + 'static, { let db = { let state = state.borrow(); - let resource = - state.resource_table.get::>(rid)?; + let resource = state + .resource_table + .get::>(rid) + .map_err(KvErrorKind::Resource)?; resource.db.clone() }; @@ -278,10 +343,7 @@ where }; if ranges.len() > config.max_read_ranges { - return Err(type_error(format!( - "Too many ranges (max {})", - config.max_read_ranges - ))); + return Err(KvErrorKind::TooManyRanges(config.max_read_ranges).into_box()); } let mut total_entries = 0usize; @@ -300,33 +362,34 @@ where Ok(ReadRange { start, end, - limit: NonZeroU32::new(limit) - .with_context(|| "limit must be greater than 0")?, + limit: NonZeroU32::new(limit).ok_or(KvErrorKind::InvalidLimit)?, reverse, }) }) - .collect::, AnyError>>()?; + .collect::, KvError>>()?; if total_entries > config.max_read_entries { - return Err(type_error(format!( - "Too many entries (max {})", - config.max_read_entries - ))); + return Err( + KvErrorKind::TooManyEntries(config.max_read_entries).into_box(), + ); } let opts = SnapshotReadOptions { consistency: consistency.into(), }; - let output_ranges = db.snapshot_read(read_ranges, opts).await?; + let output_ranges = db + .snapshot_read(read_ranges, opts) + .await + .map_err(KvErrorKind::Kv)?; let output_ranges = output_ranges .into_iter() .map(|x| { x.entries .into_iter() .map(TryInto::try_into) - .collect::, AnyError>>() + .collect::, std::io::Error>>() }) - .collect::, AnyError>>()?; + .collect::, std::io::Error>>()?; Ok(output_ranges) } @@ -345,7 +408,7 @@ impl Resource for QueueMessageResource { async fn op_kv_dequeue_next_message( state: Rc>, #[smi] rid: ResourceId, -) -> Result, AnyError> +) -> Result, KvError> where DBH: DatabaseHandler + 'static, { @@ -358,17 +421,19 @@ where if get_custom_error_class(&err) == Some("BadResource") { return Ok(None); } else { - return Err(err); + return Err(KvErrorKind::Resource(err).into_box()); } } }; resource.db.clone() }; - let Some(mut handle) = db.dequeue_next_message().await? else { + let Some(mut handle) = + db.dequeue_next_message().await.map_err(KvErrorKind::Kv)? + else { return Ok(None); }; - let payload = handle.take_payload().await?.into(); + let payload = handle.take_payload().await.map_err(KvErrorKind::Kv)?.into(); let handle_rid = { let mut state = state.borrow_mut(); state.resource_table.add(QueueMessageResource { handle }) @@ -382,18 +447,18 @@ fn op_kv_watch( state: &mut OpState, #[smi] rid: ResourceId, #[serde] keys: Vec, -) -> Result +) -> Result where DBH: DatabaseHandler + 'static, { - let resource = state.resource_table.get::>(rid)?; + let resource = state + .resource_table + .get::>(rid) + .map_err(KvErrorKind::Resource)?; let config = state.borrow::>().clone(); if keys.len() > config.max_watched_keys { - return Err(type_error(format!( - "Too many keys (max {})", - config.max_watched_keys - ))); + return Err(KvErrorKind::TooManyKeys(config.max_watched_keys).into_box()); } let keys: Vec> = keys @@ -428,10 +493,13 @@ enum WatchEntry { async fn op_kv_watch_next( state: Rc>, #[smi] rid: ResourceId, -) -> Result>, AnyError> { +) -> Result>, KvError> { let resource = { let state = state.borrow(); - let resource = state.resource_table.get::(rid)?; + let resource = state + .resource_table + .get::(rid) + .map_err(KvErrorKind::Resource)?; resource.clone() }; @@ -457,7 +525,7 @@ async fn op_kv_watch_next( return Ok(None); }; - let entries = res?; + let entries = res.map_err(KvErrorKind::Kv)?; let entries = entries .into_iter() .map(|entry| { @@ -468,7 +536,7 @@ async fn op_kv_watch_next( WatchKeyOutput::Unchanged => WatchEntry::Unchanged, }) }) - .collect::>()?; + .collect::>()?; Ok(Some(entries)) } @@ -478,7 +546,7 @@ async fn op_kv_finish_dequeued_message( state: Rc>, #[smi] handle_rid: ResourceId, success: bool, -) -> Result<(), AnyError> +) -> Result<(), KvError> where DBH: DatabaseHandler + 'static, { @@ -487,9 +555,9 @@ where let handle = state .resource_table .take::::DB as Database>::QMH>>(handle_rid) - .map_err(|_| type_error("Queue message not found"))?; + .map_err(|_| KvErrorKind::QueueMessageNotFound)?; Rc::try_unwrap(handle) - .map_err(|_| type_error("Queue message not found"))? + .map_err(|_| KvErrorKind::QueueMessageNotFound)? .handle }; // if we fail to finish the message, there is not much we can do and the @@ -500,32 +568,52 @@ where Ok(()) } +#[derive(Debug, thiserror::Error)] +pub enum KvCheckError { + #[error("invalid versionstamp")] + InvalidVersionstamp, + #[error(transparent)] + Io(std::io::Error), +} + type V8KvCheck = (KvKey, Option); -fn check_from_v8(value: V8KvCheck) -> Result { +fn check_from_v8(value: V8KvCheck) -> Result { let versionstamp = match value.1 { Some(data) => { let mut out = [0u8; 10]; if data.len() != out.len() * 2 { - bail!(type_error("invalid versionstamp")); + return Err(KvCheckError::InvalidVersionstamp); } faster_hex::hex_decode(&data, &mut out) - .map_err(|_| type_error("invalid versionstamp"))?; + .map_err(|_| KvCheckError::InvalidVersionstamp)?; Some(out) } None => None, }; Ok(Check { - key: encode_v8_key(value.0)?, + key: encode_v8_key(value.0).map_err(KvCheckError::Io)?, versionstamp, }) } +#[derive(Debug, thiserror::Error)] +pub enum KvMutationError { + #[error(transparent)] + BigInt(#[from] num_bigint::TryFromBigIntError), + #[error(transparent)] + Io(#[from] std::io::Error), + #[error("Invalid mutation '{0}' with value")] + InvalidMutationWithValue(String), + #[error("Invalid mutation '{0}' without value")] + InvalidMutationWithoutValue(String), +} + type V8KvMutation = (KvKey, String, Option, Option); fn mutation_from_v8( (value, current_timstamp): (V8KvMutation, DateTime), -) -> Result { +) -> Result { let key = encode_v8_key(value.0)?; let kind = match (value.1.as_str(), value.2) { ("set", Some(value)) => MutationKind::Set(value.try_into()?), @@ -542,10 +630,10 @@ fn mutation_from_v8( MutationKind::SetSuffixVersionstampedKey(value.try_into()?) } (op, Some(_)) => { - return Err(type_error(format!("Invalid mutation '{op}' with value"))) + return Err(KvMutationError::InvalidMutationWithValue(op.to_string())) } (op, None) => { - return Err(type_error(format!("Invalid mutation '{op}' without value"))) + return Err(KvMutationError::InvalidMutationWithoutValue(op.to_string())) } }; Ok(Mutation { @@ -562,7 +650,7 @@ type V8Enqueue = (JsBuffer, u64, Vec, Option>); fn enqueue_from_v8( value: V8Enqueue, current_timestamp: DateTime, -) -> Result { +) -> Result { Ok(Enqueue { payload: value.0.to_vec(), deadline: current_timestamp @@ -597,7 +685,7 @@ impl RawSelector { prefix: Option, start: Option, end: Option, - ) -> Result { + ) -> Result { let prefix = prefix.map(encode_v8_key).transpose()?; let start = start.map(encode_v8_key).transpose()?; let end = end.map(encode_v8_key).transpose()?; @@ -610,9 +698,7 @@ impl RawSelector { }), (Some(prefix), Some(start), None) => { if !start.starts_with(&prefix) || start.len() == prefix.len() { - return Err(type_error( - "Start key is not in the keyspace defined by prefix", - )); + return Err(KvErrorKind::StartKeyNotInKeyspace.into_box()); } Ok(Self::Prefixed { prefix, @@ -622,9 +708,7 @@ impl RawSelector { } (Some(prefix), None, Some(end)) => { if !end.starts_with(&prefix) || end.len() == prefix.len() { - return Err(type_error( - "End key is not in the keyspace defined by prefix", - )); + return Err(KvErrorKind::EndKeyNotInKeyspace.into_box()); } Ok(Self::Prefixed { prefix, @@ -634,7 +718,7 @@ impl RawSelector { } (None, Some(start), Some(end)) => { if start > end { - return Err(type_error("Start key is greater than end key")); + return Err(KvErrorKind::StartKeyGreaterThanEndKey.into_box()); } Ok(Self::Range { start, end }) } @@ -642,7 +726,7 @@ impl RawSelector { let end = start.iter().copied().chain(Some(0)).collect(); Ok(Self::Range { start, end }) } - _ => Err(type_error("Invalid range")), + _ => Err(KvErrorKind::InvalidRange.into_box()), } } @@ -701,10 +785,10 @@ fn common_prefix_for_bytes<'a>(a: &'a [u8], b: &'a [u8]) -> &'a [u8] { fn encode_cursor( selector: &RawSelector, boundary_key: &[u8], -) -> Result { +) -> Result { let common_prefix = selector.common_prefix(); if !boundary_key.starts_with(common_prefix) { - return Err(type_error("Invalid boundary key")); + return Err(KvErrorKind::InvalidBoundaryKey.into_box()); } Ok(BASE64_URL_SAFE.encode(&boundary_key[common_prefix.len()..])) } @@ -713,7 +797,7 @@ fn decode_selector_and_cursor( selector: &RawSelector, reverse: bool, cursor: Option<&ByteString>, -) -> Result<(Vec, Vec), AnyError> { +) -> Result<(Vec, Vec), KvError> { let Some(cursor) = cursor else { return Ok((selector.range_start_key(), selector.range_end_key())); }; @@ -721,7 +805,7 @@ fn decode_selector_and_cursor( let common_prefix = selector.common_prefix(); let cursor = BASE64_URL_SAFE .decode(cursor) - .map_err(|_| type_error("invalid cursor"))?; + .map_err(|_| KvErrorKind::InvalidCursor)?; let first_key: Vec; let last_key: Vec; @@ -746,13 +830,13 @@ fn decode_selector_and_cursor( // Defend against out-of-bounds reading if let Some(start) = selector.start() { if &first_key[..] < start { - return Err(type_error("cursor out of bounds")); + return Err(KvErrorKind::CursorOutOfBounds.into_box()); } } if let Some(end) = selector.end() { if &last_key[..] > end { - return Err(type_error("cursor out of bounds")); + return Err(KvErrorKind::CursorOutOfBounds.into_box()); } } @@ -767,15 +851,17 @@ async fn op_kv_atomic_write( #[serde] checks: Vec, #[serde] mutations: Vec, #[serde] enqueues: Vec, -) -> Result, AnyError> +) -> Result, KvError> where DBH: DatabaseHandler + 'static, { let current_timestamp = chrono::Utc::now(); let db = { let state = state.borrow(); - let resource = - state.resource_table.get::>(rid)?; + let resource = state + .resource_table + .get::>(rid) + .map_err(KvErrorKind::Resource)?; resource.db.clone() }; @@ -785,34 +871,28 @@ where }; if checks.len() > config.max_checks { - return Err(type_error(format!( - "Too many checks (max {})", - config.max_checks - ))); + return Err(KvErrorKind::TooManyChecks(config.max_checks).into_box()); } if mutations.len() + enqueues.len() > config.max_mutations { - return Err(type_error(format!( - "Too many mutations (max {})", - config.max_mutations - ))); + return Err(KvErrorKind::TooManyMutations(config.max_mutations).into_box()); } let checks = checks .into_iter() .map(check_from_v8) - .collect::, AnyError>>() - .with_context(|| "invalid check")?; + .collect::, KvCheckError>>() + .map_err(KvErrorKind::InvalidCheck)?; let mutations = mutations .into_iter() .map(|mutation| mutation_from_v8((mutation, current_timestamp))) - .collect::, AnyError>>() - .with_context(|| "Invalid mutation")?; + .collect::, KvMutationError>>() + .map_err(KvErrorKind::InvalidMutation)?; let enqueues = enqueues .into_iter() .map(|e| enqueue_from_v8(e, current_timestamp)) - .collect::, AnyError>>() - .with_context(|| "invalid enqueue")?; + .collect::, std::io::Error>>() + .map_err(KvErrorKind::InvalidEnqueue)?; let mut total_payload_size = 0usize; let mut total_key_size = 0usize; @@ -823,7 +903,7 @@ where .chain(mutations.iter().map(|m| &m.key)) { if key.is_empty() { - return Err(type_error("key cannot be empty")); + return Err(KvErrorKind::EmptyKey.into_box()); } total_payload_size += check_write_key_size(key, &config)?; @@ -847,17 +927,16 @@ where } if total_payload_size > config.max_total_mutation_size_bytes { - return Err(type_error(format!( - "Total mutation size too large (max {} bytes)", - config.max_total_mutation_size_bytes - ))); + return Err( + KvErrorKind::TotalMutationTooLarge(config.max_total_mutation_size_bytes) + .into_box(), + ); } if total_key_size > config.max_total_key_size_bytes { - return Err(type_error(format!( - "Total key size too large (max {} bytes)", - config.max_total_key_size_bytes - ))); + return Err( + KvErrorKind::TotalKeyTooLarge(config.max_total_key_size_bytes).into_box(), + ); } let atomic_write = AtomicWrite { @@ -866,7 +945,10 @@ where enqueues, }; - let result = db.atomic_write(atomic_write).await?; + let result = db + .atomic_write(atomic_write) + .await + .map_err(KvErrorKind::Kv)?; Ok(result.map(|res| faster_hex::hex_string(&res.versionstamp))) } @@ -879,19 +961,18 @@ type EncodeCursorRangeSelector = (Option, Option, Option); fn op_kv_encode_cursor( #[serde] (prefix, start, end): EncodeCursorRangeSelector, #[serde] boundary_key: KvKey, -) -> Result { +) -> Result { let selector = RawSelector::from_tuple(prefix, start, end)?; let boundary_key = encode_v8_key(boundary_key)?; let cursor = encode_cursor(&selector, &boundary_key)?; Ok(cursor) } -fn check_read_key_size(key: &[u8], config: &KvConfig) -> Result<(), AnyError> { +fn check_read_key_size(key: &[u8], config: &KvConfig) -> Result<(), KvError> { if key.len() > config.max_read_key_size_bytes { - Err(type_error(format!( - "Key too large for read (max {} bytes)", - config.max_read_key_size_bytes - ))) + Err( + KvErrorKind::KeyTooLargeToRead(config.max_read_key_size_bytes).into_box(), + ) } else { Ok(()) } @@ -900,12 +981,12 @@ fn check_read_key_size(key: &[u8], config: &KvConfig) -> Result<(), AnyError> { fn check_write_key_size( key: &[u8], config: &KvConfig, -) -> Result { +) -> Result { if key.len() > config.max_write_key_size_bytes { - Err(type_error(format!( - "Key too large for write (max {} bytes)", - config.max_write_key_size_bytes - ))) + Err( + KvErrorKind::KeyTooLargeToWrite(config.max_write_key_size_bytes) + .into_box(), + ) } else { Ok(key.len()) } @@ -914,7 +995,7 @@ fn check_write_key_size( fn check_value_size( value: &KvValue, config: &KvConfig, -) -> Result { +) -> Result { let payload = match value { KvValue::Bytes(x) => x, KvValue::V8(x) => x, @@ -922,10 +1003,7 @@ fn check_value_size( }; if payload.len() > config.max_value_size_bytes { - Err(type_error(format!( - "Value too large (max {} bytes)", - config.max_value_size_bytes - ))) + Err(KvErrorKind::ValueTooLarge(config.max_value_size_bytes).into_box()) } else { Ok(payload.len()) } @@ -934,12 +1012,12 @@ fn check_value_size( fn check_enqueue_payload_size( payload: &[u8], config: &KvConfig, -) -> Result { +) -> Result { if payload.len() > config.max_value_size_bytes { - Err(type_error(format!( - "enqueue payload too large (max {} bytes)", - config.max_value_size_bytes - ))) + Err( + KvErrorKind::EnqueuePayloadTooLarge(config.max_value_size_bytes) + .into_box(), + ) } else { Ok(payload.len()) } diff --git a/ext/kv/remote.rs b/ext/kv/remote.rs index 922853588addb0..1830aa67ee6df3 100644 --- a/ext/kv/remote.rs +++ b/ext/kv/remote.rs @@ -15,6 +15,7 @@ use deno_core::futures::Stream; use deno_core::OpState; use deno_fetch::create_http_client; use deno_fetch::CreateHttpClientOptions; +use deno_permissions::PermissionCheckError; use deno_tls::rustls::RootCertStore; use deno_tls::Proxy; use deno_tls::RootCertStoreProvider; @@ -45,17 +46,17 @@ impl HttpOptions { } pub trait RemoteDbHandlerPermissions { - fn check_env(&mut self, var: &str) -> Result<(), AnyError>; + fn check_env(&mut self, var: &str) -> Result<(), PermissionCheckError>; fn check_net_url( &mut self, url: &Url, api_name: &str, - ) -> Result<(), AnyError>; + ) -> Result<(), PermissionCheckError>; } impl RemoteDbHandlerPermissions for deno_permissions::PermissionsContainer { #[inline(always)] - fn check_env(&mut self, var: &str) -> Result<(), AnyError> { + fn check_env(&mut self, var: &str) -> Result<(), PermissionCheckError> { deno_permissions::PermissionsContainer::check_env(self, var) } @@ -64,7 +65,7 @@ impl RemoteDbHandlerPermissions for deno_permissions::PermissionsContainer { &mut self, url: &Url, api_name: &str, - ) -> Result<(), AnyError> { + ) -> Result<(), PermissionCheckError> { deno_permissions::PermissionsContainer::check_net_url(self, url, api_name) } } @@ -103,7 +104,9 @@ impl denokv_remote::RemotePermissions fn check_net_url(&self, url: &Url) -> Result<(), anyhow::Error> { let mut state = self.state.borrow_mut(); let permissions = state.borrow_mut::

(); - permissions.check_net_url(url, "Deno.openKv") + permissions + .check_net_url(url, "Deno.openKv") + .map_err(Into::into) } } @@ -194,6 +197,7 @@ impl DatabaseHandler root_cert_store: options.root_cert_store()?, ca_certs: vec![], proxy: options.proxy.clone(), + dns_resolver: Default::default(), unsafely_ignore_certificate_errors: options .unsafely_ignore_certificate_errors .clone(), @@ -206,6 +210,7 @@ impl DatabaseHandler pool_idle_timeout: None, http1: false, http2: true, + client_builder_hook: None, }, )?; let fetch_client = FetchClient(client); diff --git a/ext/kv/sqlite.rs b/ext/kv/sqlite.rs index 8027ff03d4c09c..9de5209275c28c 100644 --- a/ext/kv/sqlite.rs +++ b/ext/kv/sqlite.rs @@ -13,20 +13,20 @@ use std::sync::Arc; use std::sync::Mutex; use std::sync::OnceLock; +use crate::DatabaseHandler; use async_trait::async_trait; use deno_core::error::type_error; use deno_core::error::AnyError; -use deno_core::normalize_path; use deno_core::unsync::spawn_blocking; use deno_core::OpState; +use deno_path_util::normalize_path; +use deno_permissions::PermissionCheckError; pub use denokv_sqlite::SqliteBackendError; use denokv_sqlite::SqliteConfig; use denokv_sqlite::SqliteNotifier; use rand::SeedableRng; use rusqlite::OpenFlags; -use crate::DatabaseHandler; - static SQLITE_NOTIFIERS_MAP: OnceLock>> = OnceLock::new(); @@ -42,13 +42,13 @@ pub trait SqliteDbHandlerPermissions { &mut self, p: &str, api_name: &str, - ) -> Result; + ) -> Result; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] fn check_write<'a>( &mut self, p: &'a Path, api_name: &str, - ) -> Result, AnyError>; + ) -> Result, PermissionCheckError>; } impl SqliteDbHandlerPermissions for deno_permissions::PermissionsContainer { @@ -57,7 +57,7 @@ impl SqliteDbHandlerPermissions for deno_permissions::PermissionsContainer { &mut self, p: &str, api_name: &str, - ) -> Result { + ) -> Result { deno_permissions::PermissionsContainer::check_read(self, p, api_name) } @@ -66,7 +66,7 @@ impl SqliteDbHandlerPermissions for deno_permissions::PermissionsContainer { &mut self, p: &'a Path, api_name: &str, - ) -> Result, AnyError> { + ) -> Result, PermissionCheckError> { deno_permissions::PermissionsContainer::check_write_path(self, p, api_name) } } diff --git a/ext/napi/Cargo.toml b/ext/napi/Cargo.toml index aabf6d31f5c9dc..0910aacc7f0689 100644 --- a/ext/napi/Cargo.toml +++ b/ext/napi/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_napi" -version = "0.99.0" +version = "0.111.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -16,4 +16,14 @@ path = "lib.rs" [dependencies] deno_core.workspace = true deno_permissions.workspace = true +libc.workspace = true libloading = { version = "0.7" } +log.workspace = true +napi_sym.workspace = true +thiserror.workspace = true + +[target.'cfg(windows)'.dependencies] +windows-sys.workspace = true + +[dev-dependencies] +libuv-sys-lite = "=1.48.2" diff --git a/ext/napi/README.md b/ext/napi/README.md index e69de29bb2d1d6..b4792952444d20 100644 --- a/ext/napi/README.md +++ b/ext/napi/README.md @@ -0,0 +1,114 @@ +# napi + +This directory contains source for Deno's Node-API implementation. It depends on +`napi_sym` and `deno_napi`. + +Files are generally organized the same as in Node.js's implementation to ease in +ensuring compatibility. + +## Adding a new function + +Add the symbol name to +[`cli/napi_sym/symbol_exports.json`](../napi_sym/symbol_exports.json). + +```diff +{ + "symbols": [ + ... + "napi_get_undefined", +- "napi_get_null" ++ "napi_get_null", ++ "napi_get_boolean" + ] +} +``` + +Determine where to place the implementation. `napi_get_boolean` is related to JS +values so we will place it in `js_native_api.rs`. If something is not clear, +just create a new file module. + +See [`napi_sym`](../napi_sym/) for writing the implementation: + +```rust +#[napi_sym::napi_sym] +fn napi_get_boolean( + env: *mut Env, + value: bool, + result: *mut napi_value, +) -> Result { + // ... + Ok(()) +} +``` + +Update the generated symbol lists using the script: + +``` +deno run --allow-write tools/napi/generate_symbols_lists.js +``` + +Add a test in [`/tests/napi`](../../tests/napi/). You can also refer to Node.js +test suite for Node-API. + +```js +// tests/napi/boolean_test.js +import { assertEquals, loadTestLibrary } from "./common.js"; +const lib = loadTestLibrary(); +Deno.test("napi get boolean", function () { + assertEquals(lib.test_get_boolean(true), true); + assertEquals(lib.test_get_boolean(false), false); +}); +``` + +```rust +// tests/napi/src/boolean.rs + +use napi_sys::Status::napi_ok; +use napi_sys::ValueType::napi_boolean; +use napi_sys::*; + +extern "C" fn test_boolean( + env: napi_env, + info: napi_callback_info, +) -> napi_value { + let (args, argc, _) = crate::get_callback_info!(env, info, 1); + assert_eq!(argc, 1); + + let mut ty = -1; + assert!(unsafe { napi_typeof(env, args[0], &mut ty) } == napi_ok); + assert_eq!(ty, napi_boolean); + + // Use napi_get_boolean here... + + value +} + +pub fn init(env: napi_env, exports: napi_value) { + let properties = &[crate::new_property!(env, "test_boolean\0", test_boolean)]; + + unsafe { + napi_define_properties(env, exports, properties.len(), properties.as_ptr()) + }; +} +``` + +```diff +// tests/napi/src/lib.rs + ++ mod boolean; + +... + +#[no_mangle] +unsafe extern "C" fn napi_register_module_v1( + env: napi_env, + exports: napi_value, +) -> napi_value { + ... ++ boolean::init(env, exports); + + exports +} +``` + +Run the test using `cargo test -p tests/napi`. diff --git a/ext/napi/build.rs b/ext/napi/build.rs new file mode 100644 index 00000000000000..8705830a95c4da --- /dev/null +++ b/ext/napi/build.rs @@ -0,0 +1,22 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +fn main() { + let symbols_file_name = match std::env::consts::OS { + "android" | "freebsd" | "openbsd" => { + "generated_symbol_exports_list_linux.def".to_string() + } + os => format!("generated_symbol_exports_list_{}.def", os), + }; + let symbols_path = std::path::Path::new(".") + .join(symbols_file_name) + .canonicalize() + .expect( + "Missing symbols list! Generate using tools/napi/generate_symbols_lists.js", + ); + + println!("cargo:rustc-rerun-if-changed={}", symbols_path.display()); + + let path = std::path::PathBuf::from(std::env::var("OUT_DIR").unwrap()) + .join("napi_symbol_path.txt"); + std::fs::write(path, symbols_path.as_os_str().as_encoded_bytes()).unwrap(); +} diff --git a/cli/napi/generated_symbol_exports_list_linux.def b/ext/napi/generated_symbol_exports_list_linux.def similarity index 96% rename from cli/napi/generated_symbol_exports_list_linux.def rename to ext/napi/generated_symbol_exports_list_linux.def index 06e94f05bbf490..614880ebfa9da6 100644 --- a/cli/napi/generated_symbol_exports_list_linux.def +++ b/ext/napi/generated_symbol_exports_list_linux.def @@ -1 +1 @@ -{ "node_api_create_syntax_error"; "napi_make_callback"; "napi_has_named_property"; "napi_async_destroy"; "napi_coerce_to_object"; "napi_get_arraybuffer_info"; "napi_detach_arraybuffer"; "napi_get_undefined"; "napi_reference_unref"; "napi_fatal_error"; "napi_open_callback_scope"; "napi_close_callback_scope"; "napi_get_value_uint32"; "napi_create_function"; "napi_create_arraybuffer"; "napi_get_value_int64"; "napi_get_all_property_names"; "napi_resolve_deferred"; "napi_is_detached_arraybuffer"; "napi_create_string_utf8"; "napi_create_threadsafe_function"; "node_api_throw_syntax_error"; "napi_create_bigint_int64"; "napi_wrap"; "napi_set_property"; "napi_get_value_bigint_int64"; "napi_open_handle_scope"; "napi_create_error"; "napi_create_buffer"; "napi_cancel_async_work"; "napi_is_exception_pending"; "napi_acquire_threadsafe_function"; "napi_create_external"; "napi_get_threadsafe_function_context"; "napi_get_null"; "napi_create_string_utf16"; "node_api_create_external_string_utf16"; "napi_get_value_bigint_uint64"; "napi_module_register"; "napi_is_typedarray"; "napi_create_external_buffer"; "napi_get_new_target"; "napi_get_instance_data"; "napi_close_handle_scope"; "napi_get_value_string_utf16"; "napi_get_property_names"; "napi_is_arraybuffer"; "napi_get_cb_info"; "napi_define_properties"; "napi_add_env_cleanup_hook"; "node_api_get_module_file_name"; "napi_get_node_version"; "napi_create_int64"; "napi_create_double"; "napi_get_and_clear_last_exception"; "napi_create_reference"; "napi_get_typedarray_info"; "napi_call_threadsafe_function"; "napi_get_last_error_info"; "napi_create_array_with_length"; "napi_coerce_to_number"; "napi_get_global"; "napi_is_error"; "napi_set_instance_data"; "napi_create_typedarray"; "napi_throw_type_error"; "napi_has_property"; "napi_get_value_external"; "napi_create_range_error"; "napi_typeof"; "napi_ref_threadsafe_function"; "napi_create_bigint_uint64"; "napi_get_prototype"; "napi_adjust_external_memory"; "napi_release_threadsafe_function"; "napi_delete_async_work"; "napi_create_string_latin1"; "node_api_create_external_string_latin1"; "napi_is_array"; "napi_unref_threadsafe_function"; "napi_throw_error"; "napi_has_own_property"; "napi_get_reference_value"; "napi_remove_env_cleanup_hook"; "napi_get_value_string_utf8"; "napi_is_promise"; "napi_get_boolean"; "napi_run_script"; "napi_get_element"; "napi_get_named_property"; "napi_get_buffer_info"; "napi_get_value_bool"; "napi_reference_ref"; "napi_create_object"; "napi_create_promise"; "napi_create_int32"; "napi_escape_handle"; "napi_open_escapable_handle_scope"; "napi_throw"; "napi_get_value_double"; "napi_set_named_property"; "napi_call_function"; "napi_create_date"; "napi_object_freeze"; "napi_get_uv_event_loop"; "napi_get_value_string_latin1"; "napi_reject_deferred"; "napi_add_finalizer"; "napi_create_array"; "napi_delete_reference"; "napi_get_date_value"; "napi_create_dataview"; "napi_get_version"; "napi_define_class"; "napi_is_date"; "napi_remove_wrap"; "napi_delete_property"; "napi_instanceof"; "napi_create_buffer_copy"; "napi_delete_element"; "napi_object_seal"; "napi_queue_async_work"; "napi_get_value_bigint_words"; "napi_is_buffer"; "napi_get_array_length"; "napi_get_property"; "napi_new_instance"; "napi_set_element"; "napi_create_bigint_words"; "napi_strict_equals"; "napi_is_dataview"; "napi_close_escapable_handle_scope"; "napi_get_dataview_info"; "napi_get_value_int32"; "napi_unwrap"; "napi_throw_range_error"; "napi_coerce_to_bool"; "napi_create_uint32"; "napi_has_element"; "napi_create_external_arraybuffer"; "napi_create_symbol"; "node_api_symbol_for"; "napi_coerce_to_string"; "napi_create_type_error"; "napi_fatal_exception"; "napi_create_async_work"; "napi_async_init"; "node_api_create_property_key_utf16"; "napi_type_tag_object"; "napi_check_object_type_tag"; "node_api_post_finalizer"; "napi_add_async_cleanup_hook"; "napi_remove_async_cleanup_hook"; }; \ No newline at end of file +{ "node_api_create_syntax_error"; "napi_make_callback"; "napi_has_named_property"; "napi_async_destroy"; "napi_coerce_to_object"; "napi_get_arraybuffer_info"; "napi_detach_arraybuffer"; "napi_get_undefined"; "napi_reference_unref"; "napi_fatal_error"; "napi_open_callback_scope"; "napi_close_callback_scope"; "napi_get_value_uint32"; "napi_create_function"; "napi_create_arraybuffer"; "napi_get_value_int64"; "napi_get_all_property_names"; "napi_resolve_deferred"; "napi_is_detached_arraybuffer"; "napi_create_string_utf8"; "napi_create_threadsafe_function"; "node_api_throw_syntax_error"; "napi_create_bigint_int64"; "napi_wrap"; "napi_set_property"; "napi_get_value_bigint_int64"; "napi_open_handle_scope"; "napi_create_error"; "napi_create_buffer"; "napi_cancel_async_work"; "napi_is_exception_pending"; "napi_acquire_threadsafe_function"; "napi_create_external"; "napi_get_threadsafe_function_context"; "napi_get_null"; "napi_create_string_utf16"; "node_api_create_external_string_utf16"; "napi_get_value_bigint_uint64"; "napi_module_register"; "napi_is_typedarray"; "napi_create_external_buffer"; "napi_get_new_target"; "napi_get_instance_data"; "napi_close_handle_scope"; "napi_get_value_string_utf16"; "napi_get_property_names"; "napi_is_arraybuffer"; "napi_get_cb_info"; "napi_define_properties"; "napi_add_env_cleanup_hook"; "node_api_get_module_file_name"; "napi_get_node_version"; "napi_create_int64"; "napi_create_double"; "napi_get_and_clear_last_exception"; "napi_create_reference"; "napi_get_typedarray_info"; "napi_call_threadsafe_function"; "napi_get_last_error_info"; "napi_create_array_with_length"; "napi_coerce_to_number"; "napi_get_global"; "napi_is_error"; "napi_set_instance_data"; "napi_create_typedarray"; "napi_throw_type_error"; "napi_has_property"; "napi_get_value_external"; "napi_create_range_error"; "napi_typeof"; "napi_ref_threadsafe_function"; "napi_create_bigint_uint64"; "napi_get_prototype"; "napi_adjust_external_memory"; "napi_release_threadsafe_function"; "napi_delete_async_work"; "napi_create_string_latin1"; "node_api_create_external_string_latin1"; "napi_is_array"; "napi_unref_threadsafe_function"; "napi_throw_error"; "napi_has_own_property"; "napi_get_reference_value"; "napi_remove_env_cleanup_hook"; "napi_get_value_string_utf8"; "napi_is_promise"; "napi_get_boolean"; "napi_run_script"; "napi_get_element"; "napi_get_named_property"; "napi_get_buffer_info"; "napi_get_value_bool"; "napi_reference_ref"; "napi_create_object"; "napi_create_promise"; "napi_create_int32"; "napi_escape_handle"; "napi_open_escapable_handle_scope"; "napi_throw"; "napi_get_value_double"; "napi_set_named_property"; "napi_call_function"; "napi_create_date"; "napi_object_freeze"; "napi_get_uv_event_loop"; "napi_get_value_string_latin1"; "napi_reject_deferred"; "napi_add_finalizer"; "napi_create_array"; "napi_delete_reference"; "napi_get_date_value"; "napi_create_dataview"; "napi_get_version"; "napi_define_class"; "napi_is_date"; "napi_remove_wrap"; "napi_delete_property"; "napi_instanceof"; "napi_create_buffer_copy"; "napi_delete_element"; "napi_object_seal"; "napi_queue_async_work"; "napi_get_value_bigint_words"; "napi_is_buffer"; "napi_get_array_length"; "napi_get_property"; "napi_new_instance"; "napi_set_element"; "napi_create_bigint_words"; "napi_strict_equals"; "napi_is_dataview"; "napi_close_escapable_handle_scope"; "napi_get_dataview_info"; "napi_get_value_int32"; "napi_unwrap"; "napi_throw_range_error"; "napi_coerce_to_bool"; "napi_create_uint32"; "napi_has_element"; "napi_create_external_arraybuffer"; "napi_create_symbol"; "node_api_symbol_for"; "napi_coerce_to_string"; "napi_create_type_error"; "napi_fatal_exception"; "napi_create_async_work"; "napi_async_init"; "node_api_create_property_key_utf16"; "napi_type_tag_object"; "napi_check_object_type_tag"; "node_api_post_finalizer"; "napi_add_async_cleanup_hook"; "napi_remove_async_cleanup_hook"; "uv_mutex_init"; "uv_mutex_lock"; "uv_mutex_unlock"; "uv_mutex_destroy"; "uv_async_init"; "uv_async_send"; "uv_close"; }; \ No newline at end of file diff --git a/cli/napi/generated_symbol_exports_list_macos.def b/ext/napi/generated_symbol_exports_list_macos.def similarity index 96% rename from cli/napi/generated_symbol_exports_list_macos.def rename to ext/napi/generated_symbol_exports_list_macos.def index cac7100c6fdf4c..36b2f37fa5613b 100644 --- a/cli/napi/generated_symbol_exports_list_macos.def +++ b/ext/napi/generated_symbol_exports_list_macos.def @@ -150,4 +150,11 @@ _napi_type_tag_object _napi_check_object_type_tag _node_api_post_finalizer _napi_add_async_cleanup_hook -_napi_remove_async_cleanup_hook \ No newline at end of file +_napi_remove_async_cleanup_hook +_uv_mutex_init +_uv_mutex_lock +_uv_mutex_unlock +_uv_mutex_destroy +_uv_async_init +_uv_async_send +_uv_close \ No newline at end of file diff --git a/cli/napi/generated_symbol_exports_list_windows.def b/ext/napi/generated_symbol_exports_list_windows.def similarity index 96% rename from cli/napi/generated_symbol_exports_list_windows.def rename to ext/napi/generated_symbol_exports_list_windows.def index 5386b46e54f8c1..b7355112e71c35 100644 --- a/cli/napi/generated_symbol_exports_list_windows.def +++ b/ext/napi/generated_symbol_exports_list_windows.def @@ -152,4 +152,11 @@ EXPORTS napi_check_object_type_tag node_api_post_finalizer napi_add_async_cleanup_hook - napi_remove_async_cleanup_hook \ No newline at end of file + napi_remove_async_cleanup_hook + uv_mutex_init + uv_mutex_lock + uv_mutex_unlock + uv_mutex_destroy + uv_async_init + uv_async_send + uv_close \ No newline at end of file diff --git a/cli/napi/js_native_api.rs b/ext/napi/js_native_api.rs similarity index 98% rename from cli/napi/js_native_api.rs rename to ext/napi/js_native_api.rs index e922d8c3f22cb9..53a12d6eba64ef 100644 --- a/cli/napi/js_native_api.rs +++ b/ext/napi/js_native_api.rs @@ -5,7 +5,7 @@ const NAPI_VERSION: u32 = 9; -use deno_runtime::deno_napi::*; +use crate::*; use libc::INT_MAX; use super::util::check_new_from_utf8; @@ -17,9 +17,9 @@ use super::util::napi_set_last_error; use super::util::v8_name_from_property_descriptor; use crate::check_arg; use crate::check_env; -use deno_runtime::deno_napi::function::create_function; -use deno_runtime::deno_napi::function::create_function_template; -use deno_runtime::deno_napi::function::CallbackInfo; +use crate::function::create_function; +use crate::function::create_function_template; +use crate::function::CallbackInfo; use napi_sym::napi_sym; use std::ptr::NonNull; @@ -264,6 +264,16 @@ fn napi_define_class<'s>( Err(status) => return status, }; + let mut accessor_property = v8::PropertyAttribute::NONE; + + if p.attributes & napi_enumerable == 0 { + accessor_property = accessor_property | v8::PropertyAttribute::DONT_ENUM; + } + if p.attributes & napi_configurable == 0 { + accessor_property = + accessor_property | v8::PropertyAttribute::DONT_DELETE; + } + if p.getter.is_some() || p.setter.is_some() { let getter = p.getter.map(|g| { create_function_template(&mut env.scope(), env_ptr, None, g, p.data) @@ -271,8 +281,6 @@ fn napi_define_class<'s>( let setter = p.setter.map(|s| { create_function_template(&mut env.scope(), env_ptr, None, s, p.data) }); - - let mut accessor_property = v8::PropertyAttribute::NONE; if getter.is_some() && setter.is_some() && (p.attributes & napi_writable) == 0 @@ -280,15 +288,6 @@ fn napi_define_class<'s>( accessor_property = accessor_property | v8::PropertyAttribute::READ_ONLY; } - if p.attributes & napi_enumerable == 0 { - accessor_property = - accessor_property | v8::PropertyAttribute::DONT_ENUM; - } - if p.attributes & napi_configurable == 0 { - accessor_property = - accessor_property | v8::PropertyAttribute::DONT_DELETE; - } - let proto = tpl.prototype_template(&mut env.scope()); proto.set_accessor_property(name, getter, setter, accessor_property); } else if let Some(method) = p.method { @@ -300,10 +299,14 @@ fn napi_define_class<'s>( p.data, ); let proto = tpl.prototype_template(&mut env.scope()); - proto.set(name, function.into()); + proto.set_with_attr(name, function.into(), accessor_property); } else { let proto = tpl.prototype_template(&mut env.scope()); - proto.set(name, p.value.unwrap().into()); + if (p.attributes & napi_writable) == 0 { + accessor_property = + accessor_property | v8::PropertyAttribute::READ_ONLY; + } + proto.set_with_attr(name, p.value.unwrap().into(), accessor_property); } } @@ -1080,7 +1083,7 @@ fn napi_create_string_latin1( } #[napi_sym] -fn napi_create_string_utf8( +pub(crate) fn napi_create_string_utf8( env_ptr: *mut Env, string: *const c_char, length: usize, @@ -1644,7 +1647,7 @@ fn napi_get_cb_info( check_arg!(env, argc); let argc = unsafe { *argc as usize }; for i in 0..argc { - let mut arg = args.get(i as _); + let arg = args.get(i as _); unsafe { *argv.add(i) = arg.into(); } @@ -1694,15 +1697,14 @@ fn napi_get_new_target( } #[napi_sym] -fn napi_call_function( - env_ptr: *mut Env, - recv: napi_value, - func: napi_value, +fn napi_call_function<'s>( + env: &'s mut Env, + recv: napi_value<'s>, + func: napi_value<'s>, argc: usize, - argv: *const napi_value, - result: *mut napi_value, + argv: *const napi_value<'s>, + result: *mut napi_value<'s>, ) -> napi_status { - let env = check_env!(env_ptr); check_arg!(env, recv); let args = if argc > 0 { check_arg!(env, argv); @@ -1716,11 +1718,11 @@ fn napi_call_function( let Some(func) = func.and_then(|f| v8::Local::::try_from(f).ok()) else { - return napi_set_last_error(env, napi_function_expected); + return napi_function_expected; }; let Some(v) = func.call(&mut env.scope(), recv.unwrap(), args) else { - return napi_set_last_error(env_ptr, napi_generic_failure); + return napi_generic_failure; }; if !result.is_null() { @@ -1729,7 +1731,7 @@ fn napi_call_function( } } - return napi_clear_last_error(env_ptr); + napi_ok } #[napi_sym] diff --git a/ext/napi/lib.rs b/ext/napi/lib.rs index 4500c66fd4dd4d..6db6af48a282df 100644 --- a/ext/napi/lib.rs +++ b/ext/napi/lib.rs @@ -5,9 +5,23 @@ #![allow(clippy::undocumented_unsafe_blocks)] #![deny(clippy::missing_safety_doc)] +//! Symbols to be exported are now defined in this JSON file. +//! The `#[napi_sym]` macro checks for missing entries and panics. +//! +//! `./tools/napi/generate_symbols_list.js` is used to generate the LINK `cli/exports.def` on Windows, +//! which is also checked into git. +//! +//! To add a new napi function: +//! 1. Place `#[napi_sym]` on top of your implementation. +//! 2. Add the function's identifier to this JSON list. +//! 3. Finally, run `tools/napi/generate_symbols_list.js` to update `ext/napi/generated_symbol_exports_list_*.def`. + +pub mod js_native_api; +pub mod node_api; +pub mod util; +pub mod uv; + use core::ptr::NonNull; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::parking_lot::RwLock; use deno_core::url::Url; @@ -20,6 +34,18 @@ use std::path::PathBuf; use std::rc::Rc; use std::thread_local; +#[derive(Debug, thiserror::Error)] +pub enum NApiError { + #[error("Invalid path")] + InvalidPath, + #[error(transparent)] + LibLoading(#[from] libloading::Error), + #[error("Unable to find register Node-API module at {}", .0.display())] + ModuleNotFound(PathBuf), + #[error(transparent)] + Permission(#[from] PermissionCheckError), +} + #[cfg(unix)] use libloading::os::unix::*; @@ -29,6 +55,7 @@ use libloading::os::windows::*; // Expose common stuff for ease of use. // `use deno_napi::*` pub use deno_core::v8; +use deno_permissions::PermissionCheckError; pub use std::ffi::CStr; pub use std::os::raw::c_char; pub use std::os::raw::c_void; @@ -482,14 +509,14 @@ deno_core::extension!(deno_napi, pub trait NapiPermissions { #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] - fn check(&mut self, path: &str) -> std::result::Result; + fn check(&mut self, path: &str) -> Result; } // NOTE(bartlomieju): for now, NAPI uses `--allow-ffi` flag, but that might // change in the future. impl NapiPermissions for deno_permissions::PermissionsContainer { #[inline(always)] - fn check(&mut self, path: &str) -> Result { + fn check(&mut self, path: &str) -> Result { deno_permissions::PermissionsContainer::check_ffi(self, path) } } @@ -503,7 +530,7 @@ static NAPI_LOADED_MODULES: std::sync::LazyLock< RwLock>, > = std::sync::LazyLock::new(|| RwLock::new(HashMap::new())); -#[op2(reentrant)] +#[op2(reentrant, stack_trace)] fn op_napi_open( scope: &mut v8::HandleScope<'scope>, isolate: *mut v8::Isolate, @@ -512,7 +539,7 @@ fn op_napi_open( global: v8::Local<'scope, v8::Object>, buffer_constructor: v8::Local<'scope, v8::Function>, report_error: v8::Local<'scope, v8::Function>, -) -> std::result::Result, AnyError> +) -> Result, NApiError> where NP: NapiPermissions + 'static, { @@ -540,7 +567,7 @@ where let type_tag = v8::Global::new(scope, type_tag); let url_filename = - Url::from_file_path(&path).map_err(|_| type_error("Invalid path"))?; + Url::from_file_path(&path).map_err(|_| NApiError::InvalidPath)?; let env_shared = EnvShared::new(napi_wrap, type_tag, format!("{url_filename}\0")); @@ -565,17 +592,11 @@ where // SAFETY: opening a DLL calls dlopen #[cfg(unix)] - let library = match unsafe { Library::open(Some(&path), flags) } { - Ok(lib) => lib, - Err(e) => return Err(type_error(e.to_string())), - }; + let library = unsafe { Library::open(Some(&path), flags) }?; // SAFETY: opening a DLL calls dlopen #[cfg(not(unix))] - let library = match unsafe { Library::load_with_flags(&path, flags) } { - Ok(lib) => lib, - Err(e) => return Err(type_error(e.to_string())), - }; + let library = unsafe { Library::load_with_flags(&path, flags) }?; let maybe_module = MODULE_TO_REGISTER.with(|cell| { let mut slot = cell.borrow_mut(); @@ -610,10 +631,7 @@ where // SAFETY: we are going blind, calling the register function on the other side. unsafe { init(env_ptr, exports.into()) } } else { - return Err(type_error(format!( - "Unable to find register Node-API module at {}", - path.display() - ))); + return Err(NApiError::ModuleNotFound(path)); }; let exports = maybe_exports.unwrap_or(exports.into()); @@ -624,3 +642,34 @@ where Ok(exports) } + +#[allow(clippy::print_stdout)] +pub fn print_linker_flags(name: &str) { + let symbols_path = + include_str!(concat!(env!("OUT_DIR"), "/napi_symbol_path.txt")); + + #[cfg(target_os = "windows")] + println!("cargo:rustc-link-arg-bin={name}=/DEF:{}", symbols_path); + + #[cfg(target_os = "macos")] + println!( + "cargo:rustc-link-arg-bin={name}=-Wl,-exported_symbols_list,{}", + symbols_path, + ); + + #[cfg(any( + target_os = "linux", + target_os = "freebsd", + target_os = "openbsd" + ))] + println!( + "cargo:rustc-link-arg-bin={name}=-Wl,--export-dynamic-symbol-list={}", + symbols_path, + ); + + #[cfg(target_os = "android")] + println!( + "cargo:rustc-link-arg-bin={name}=-Wl,--export-dynamic-symbol-list={}", + symbols_path, + ); +} diff --git a/cli/napi/node_api.rs b/ext/napi/node_api.rs similarity index 97% rename from cli/napi/node_api.rs rename to ext/napi/node_api.rs index 81cb800a7d84ca..2ca5c8d0b4fac6 100644 --- a/cli/napi/node_api.rs +++ b/ext/napi/node_api.rs @@ -9,10 +9,10 @@ use super::util::napi_set_last_error; use super::util::SendPtr; use crate::check_arg; use crate::check_env; +use crate::*; use deno_core::parking_lot::Condvar; use deno_core::parking_lot::Mutex; use deno_core::V8CrossThreadTaskSpawner; -use deno_runtime::deno_napi::*; use napi_sym::napi_sym; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicU8; @@ -140,7 +140,6 @@ fn napi_fatal_exception(env: &mut Env, err: napi_value) -> napi_status { } #[napi_sym] -#[allow(clippy::print_stderr)] fn napi_fatal_error( location: *const c_char, location_len: usize, @@ -173,9 +172,9 @@ fn napi_fatal_error( }; if let Some(location) = location { - eprintln!("NODE API FATAL ERROR: {} {}", location, message); + log::error!("NODE API FATAL ERROR: {} {}", location, message); } else { - eprintln!("NODE API FATAL ERROR: {}", message); + log::error!("NODE API FATAL ERROR: {}", message); } std::process::abort(); @@ -488,7 +487,7 @@ impl AsyncWork { } #[napi_sym] -fn napi_create_async_work( +pub(crate) fn napi_create_async_work( env: *mut Env, async_resource: napi_value, async_resource_name: napi_value, @@ -537,7 +536,10 @@ fn napi_create_async_work( } #[napi_sym] -fn napi_delete_async_work(env: *mut Env, work: napi_async_work) -> napi_status { +pub(crate) fn napi_delete_async_work( + env: *mut Env, + work: napi_async_work, +) -> napi_status { let env = check_env!(env); check_arg!(env, work); @@ -547,15 +549,23 @@ fn napi_delete_async_work(env: *mut Env, work: napi_async_work) -> napi_status { } #[napi_sym] -fn napi_get_uv_event_loop(env: *mut Env, uv_loop: *mut *mut ()) -> napi_status { - let env = check_env!(env); +fn napi_get_uv_event_loop( + env_ptr: *mut Env, + uv_loop: *mut *mut (), +) -> napi_status { + let env = check_env!(env_ptr); check_arg!(env, uv_loop); - // There is no uv_loop in Deno - napi_set_last_error(env, napi_generic_failure) + unsafe { + *uv_loop = env_ptr.cast(); + } + 0 } #[napi_sym] -fn napi_queue_async_work(env: *mut Env, work: napi_async_work) -> napi_status { +pub(crate) fn napi_queue_async_work( + env: *mut Env, + work: napi_async_work, +) -> napi_status { let env = check_env!(env); check_arg!(env, work); @@ -687,7 +697,7 @@ impl Drop for TsFn { if let Some(finalizer) = self.thread_finalize_cb { unsafe { - (finalizer)(self.env as _, self.thread_finalize_data, ptr::null_mut()); + (finalizer)(self.env as _, self.thread_finalize_data, self.context); } } } @@ -892,7 +902,7 @@ fn napi_create_threadsafe_function( }; let resource_name = resource_name.to_rust_string_lossy(&mut env.scope()); - let mut tsfn = Box::new(TsFn { + let tsfn = Box::new(TsFn { env, func, max_queue_size, diff --git a/ext/napi/sym/Cargo.toml b/ext/napi/sym/Cargo.toml new file mode 100644 index 00000000000000..ed69781fc3d38d --- /dev/null +++ b/ext/napi/sym/Cargo.toml @@ -0,0 +1,21 @@ +# Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +[package] +name = "napi_sym" +version = "0.110.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +readme = "README.md" +repository.workspace = true +description = "proc macro for writing N-API symbols" + +[lib] +path = "./lib.rs" +proc-macro = true + +[dependencies] +quote.workspace = true +serde.workspace = true +serde_json.workspace = true +syn.workspace = true diff --git a/ext/napi/sym/README.md b/ext/napi/sym/README.md new file mode 100644 index 00000000000000..66eb4bff26c1be --- /dev/null +++ b/ext/napi/sym/README.md @@ -0,0 +1,38 @@ +# napi_sym + +A proc_macro for Deno's Node-API implementation. It does the following things: + +- Marks the symbol as `#[no_mangle]` and rewrites it as + `unsafe extern "C" $name`. +- Asserts that the function symbol is present in + [`symbol_exports.json`](./symbol_exports.json). +- Maps `deno_napi::Result` to raw `napi_result`. + +```rust +use deno_napi::napi_value; +use deno_napi::Env; +use deno_napi::Error; +use deno_napi::Result; + +#[napi_sym::napi_sym] +fn napi_get_boolean( + env: *mut Env, + value: bool, + result: *mut napi_value, +) -> Result { + let _env: &mut Env = env.as_mut().ok_or(Error::InvalidArg)?; + // *result = ... + Ok(()) +} +``` + +### `symbol_exports.json` + +A file containing the symbols that need to be put into the executable's dynamic +symbol table at link-time. + +This is done using `/DEF:` on Windows, `-exported_symbol,_` on macOS and +`--export-dynamic-symbol=` on Linux. See [`cli/build.rs`](../build.rs). + +On Windows, you need to generate the `.def` file by running +[`tools/napi/generate_symbols_lists.js`](../../tools/napi/generate_symbols_lists.js). diff --git a/cli/napi/sym/lib.rs b/ext/napi/sym/lib.rs similarity index 100% rename from cli/napi/sym/lib.rs rename to ext/napi/sym/lib.rs diff --git a/cli/napi/sym/symbol_exports.json b/ext/napi/sym/symbol_exports.json similarity index 96% rename from cli/napi/sym/symbol_exports.json rename to ext/napi/sym/symbol_exports.json index 64b548d496a0d7..00946b8ed7a577 100644 --- a/cli/napi/sym/symbol_exports.json +++ b/ext/napi/sym/symbol_exports.json @@ -152,6 +152,13 @@ "napi_check_object_type_tag", "node_api_post_finalizer", "napi_add_async_cleanup_hook", - "napi_remove_async_cleanup_hook" + "napi_remove_async_cleanup_hook", + "uv_mutex_init", + "uv_mutex_lock", + "uv_mutex_unlock", + "uv_mutex_destroy", + "uv_async_init", + "uv_async_send", + "uv_close" ] } diff --git a/cli/napi/util.rs b/ext/napi/util.rs similarity index 84% rename from cli/napi/util.rs rename to ext/napi/util.rs index 63d8effbf2d908..21e9d433aad6c7 100644 --- a/cli/napi/util.rs +++ b/ext/napi/util.rs @@ -1,9 +1,9 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_runtime::deno_napi::*; +use crate::*; use libc::INT_MAX; #[repr(transparent)] -pub struct SendPtr(pub *const T); +pub(crate) struct SendPtr(pub *const T); impl SendPtr { // silly function to get around `clippy::redundant_locals` @@ -37,7 +37,7 @@ impl Drop for BufferFinalizer { } } -pub extern "C" fn backing_store_deleter_callback( +pub(crate) extern "C" fn backing_store_deleter_callback( data: *mut c_void, _byte_length: usize, deleter_data: *mut c_void, @@ -50,7 +50,7 @@ pub extern "C" fn backing_store_deleter_callback( drop(finalizer); } -pub fn make_external_backing_store( +pub(crate) fn make_external_backing_store( env: *mut Env, data: *mut c_void, byte_length: usize, @@ -90,9 +90,7 @@ macro_rules! check_env { macro_rules! return_error_status_if_false { ($env: expr, $condition: expr, $status: ident) => { if !$condition { - return Err( - $crate::napi::util::napi_set_last_error($env, $status).into(), - ); + return Err($crate::util::napi_set_last_error($env, $status).into()); } }; } @@ -101,7 +99,7 @@ macro_rules! return_error_status_if_false { macro_rules! return_status_if_false { ($env: expr, $condition: expr, $status: ident) => { if !$condition { - return $crate::napi::util::napi_set_last_error($env, $status); + return $crate::util::napi_set_last_error($env, $status); } }; } @@ -222,7 +220,7 @@ macro_rules! check_arg { ($env: expr, $ptr: expr) => { $crate::return_status_if_false!( $env, - !$crate::napi::util::Nullable::is_null(&$ptr), + !$crate::util::Nullable::is_null(&$ptr), napi_invalid_arg ); }; @@ -230,17 +228,17 @@ macro_rules! check_arg { #[macro_export] macro_rules! napi_wrap { - ( $( # $attr:tt )* fn $name:ident $( < $( $x:lifetime ),* > )? ( $env:ident : & $( $lt:lifetime )? mut Env $( , $ident:ident : $ty:ty )* $(,)? ) -> napi_status $body:block ) => { - $( # $attr )* + ( $( # [ $attr:meta ] )* $vis:vis fn $name:ident $( < $( $x:lifetime ),* > )? ( $env:ident : & $( $lt:lifetime )? mut Env $( , $ident:ident : $ty:ty )* $(,)? ) -> napi_status $body:block ) => { + $( # [ $attr ] )* #[no_mangle] - pub unsafe extern "C" fn $name $( < $( $x ),* > )? ( env_ptr : *mut Env , $( $ident : $ty ),* ) -> napi_status { + $vis unsafe extern "C" fn $name $( < $( $x ),* > )? ( env_ptr : *mut Env , $( $ident : $ty ),* ) -> napi_status { let env: & $( $lt )? mut Env = $crate::check_env!(env_ptr); if env.last_exception.is_some() { return napi_pending_exception; } - $crate::napi::util::napi_clear_last_error(env); + $crate::util::napi_clear_last_error(env); let scope_env = unsafe { &mut *env_ptr }; let scope = &mut scope_env.scope(); @@ -259,21 +257,21 @@ macro_rules! napi_wrap { let env = unsafe { &mut *env_ptr }; let global = v8::Global::new(env.isolate(), exception); env.last_exception = Some(global); - return $crate::napi::util::napi_set_last_error(env_ptr, napi_pending_exception); + return $crate::util::napi_set_last_error(env_ptr, napi_pending_exception); } if result != napi_ok { - return $crate::napi::util::napi_set_last_error(env_ptr, result); + return $crate::util::napi_set_last_error(env_ptr, result); } return result; } }; - ( $( # $attr:tt )* fn $name:ident $( < $( $x:lifetime ),* > )? ( $( $ident:ident : $ty:ty ),* $(,)? ) -> napi_status $body:block ) => { - $( # $attr )* + ( $( # [ $attr:meta ] )* $vis:vis fn $name:ident $( < $( $x:lifetime ),* > )? ( $( $ident:ident : $ty:ty ),* $(,)? ) -> napi_status $body:block ) => { + $( # [ $attr ] )* #[no_mangle] - pub unsafe extern "C" fn $name $( < $( $x ),* > )? ( $( $ident : $ty ),* ) -> napi_status { + $vis unsafe extern "C" fn $name $( < $( $x ),* > )? ( $( $ident : $ty ),* ) -> napi_status { #[inline(always)] fn inner $( < $( $x ),* > )? ( $( $ident : $ty ),* ) -> napi_status $body diff --git a/ext/napi/uv.rs b/ext/napi/uv.rs new file mode 100644 index 00000000000000..ea6b5396659a59 --- /dev/null +++ b/ext/napi/uv.rs @@ -0,0 +1,230 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use crate::*; +use deno_core::parking_lot::Mutex; +use std::mem::MaybeUninit; +use std::ptr::addr_of_mut; + +fn assert_ok(res: c_int) -> c_int { + if res != 0 { + log::error!("bad result in uv polyfill: {res}"); + // don't panic because that might unwind into + // c/c++ + std::process::abort(); + } + res +} + +use js_native_api::napi_create_string_utf8; +use node_api::napi_create_async_work; +use node_api::napi_delete_async_work; +use node_api::napi_queue_async_work; +use std::ffi::c_int; + +const UV_MUTEX_SIZE: usize = { + #[cfg(unix)] + { + std::mem::size_of::() + } + #[cfg(windows)] + { + std::mem::size_of::( + ) + } +}; + +#[repr(C)] +struct uv_mutex_t { + mutex: Mutex<()>, + _padding: [MaybeUninit; const { + (UV_MUTEX_SIZE - size_of::>()) / size_of::() + }], +} + +#[no_mangle] +unsafe extern "C" fn uv_mutex_init(lock: *mut uv_mutex_t) -> c_int { + unsafe { + addr_of_mut!((*lock).mutex).write(Mutex::new(())); + 0 + } +} + +#[no_mangle] +unsafe extern "C" fn uv_mutex_lock(lock: *mut uv_mutex_t) { + unsafe { + let guard = (*lock).mutex.lock(); + // forget the guard so it doesn't unlock when it goes out of scope. + // we're going to unlock it manually + std::mem::forget(guard); + } +} + +#[no_mangle] +unsafe extern "C" fn uv_mutex_unlock(lock: *mut uv_mutex_t) { + unsafe { + (*lock).mutex.force_unlock(); + } +} + +#[no_mangle] +unsafe extern "C" fn uv_mutex_destroy(_lock: *mut uv_mutex_t) { + // no cleanup required +} + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +#[allow(dead_code)] +enum uv_handle_type { + UV_UNKNOWN_HANDLE = 0, + UV_ASYNC, + UV_CHECK, + UV_FS_EVENT, + UV_FS_POLL, + UV_HANDLE, + UV_IDLE, + UV_NAMED_PIPE, + UV_POLL, + UV_PREPARE, + UV_PROCESS, + UV_STREAM, + UV_TCP, + UV_TIMER, + UV_TTY, + UV_UDP, + UV_SIGNAL, + UV_FILE, + UV_HANDLE_TYPE_MAX, +} + +const UV_HANDLE_SIZE: usize = 96; + +#[repr(C)] +struct uv_handle_t { + // public members + pub data: *mut c_void, + pub r#loop: *mut uv_loop_t, + pub r#type: uv_handle_type, + + _padding: [MaybeUninit; const { + (UV_HANDLE_SIZE + - size_of::<*mut c_void>() + - size_of::<*mut uv_loop_t>() + - size_of::()) + / size_of::() + }], +} + +#[cfg(unix)] +const UV_ASYNC_SIZE: usize = 128; + +#[cfg(windows)] +const UV_ASYNC_SIZE: usize = 224; + +#[repr(C)] +struct uv_async_t { + // public members + pub data: *mut c_void, + pub r#loop: *mut uv_loop_t, + pub r#type: uv_handle_type, + // private + async_cb: uv_async_cb, + work: napi_async_work, + _padding: [MaybeUninit; const { + (UV_ASYNC_SIZE + - size_of::<*mut c_void>() + - size_of::<*mut uv_loop_t>() + - size_of::() + - size_of::() + - size_of::()) + / size_of::() + }], +} + +type uv_loop_t = Env; +type uv_async_cb = extern "C" fn(handle: *mut uv_async_t); +#[no_mangle] +unsafe extern "C" fn uv_async_init( + r#loop: *mut uv_loop_t, + // probably uninitialized + r#async: *mut uv_async_t, + async_cb: uv_async_cb, +) -> c_int { + unsafe { + addr_of_mut!((*r#async).r#loop).write(r#loop); + addr_of_mut!((*r#async).r#type).write(uv_handle_type::UV_ASYNC); + addr_of_mut!((*r#async).async_cb).write(async_cb); + + let mut resource_name: MaybeUninit = MaybeUninit::uninit(); + assert_ok(napi_create_string_utf8( + r#loop, + c"uv_async".as_ptr(), + usize::MAX, + resource_name.as_mut_ptr(), + )); + let resource_name = resource_name.assume_init(); + + let res = napi_create_async_work( + r#loop, + None::>.into(), + resource_name, + Some(async_exec_wrap), + None, + r#async.cast(), + addr_of_mut!((*r#async).work), + ); + -res + } +} + +#[no_mangle] +unsafe extern "C" fn uv_async_send(handle: *mut uv_async_t) -> c_int { + unsafe { -napi_queue_async_work((*handle).r#loop, (*handle).work) } +} + +type uv_close_cb = unsafe extern "C" fn(*mut uv_handle_t); + +#[no_mangle] +unsafe extern "C" fn uv_close(handle: *mut uv_handle_t, close: uv_close_cb) { + unsafe { + if handle.is_null() { + close(handle); + return; + } + if let uv_handle_type::UV_ASYNC = (*handle).r#type { + let handle: *mut uv_async_t = handle.cast(); + napi_delete_async_work((*handle).r#loop, (*handle).work); + } + close(handle); + } +} + +unsafe extern "C" fn async_exec_wrap(_env: napi_env, data: *mut c_void) { + let data: *mut uv_async_t = data.cast(); + unsafe { + ((*data).async_cb)(data); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sizes() { + assert_eq!( + std::mem::size_of::(), + UV_MUTEX_SIZE + ); + assert_eq!( + std::mem::size_of::(), + UV_HANDLE_SIZE + ); + assert_eq!( + std::mem::size_of::(), + UV_ASYNC_SIZE + ); + assert_eq!(std::mem::size_of::(), UV_MUTEX_SIZE); + assert_eq!(std::mem::size_of::(), UV_HANDLE_SIZE); + assert_eq!(std::mem::size_of::(), UV_ASYNC_SIZE); + } +} diff --git a/ext/net/01_net.js b/ext/net/01_net.js index 5b894947ee08c2..c3e5f9e5ca8f42 100644 --- a/ext/net/01_net.js +++ b/ext/net/01_net.js @@ -194,6 +194,20 @@ class Conn { } } +class UpgradedConn extends Conn { + #rid = 0; + + constructor(rid, remoteAddr, localAddr) { + super(rid, remoteAddr, localAddr); + ObjectDefineProperty(this, internalRidSymbol, { + __proto__: null, + enumerable: false, + value: rid, + }); + this.#rid = rid; + } +} + class TcpConn extends Conn { #rid = 0; @@ -601,5 +615,6 @@ export { resolveDns, TcpConn, UnixConn, + UpgradedConn, validatePort, }; diff --git a/ext/net/02_tls.js b/ext/net/02_tls.js index 11d19440f7474d..6dad965590eb08 100644 --- a/ext/net/02_tls.js +++ b/ext/net/02_tls.js @@ -124,17 +124,19 @@ function loadTlsKeyPair(api, { // Check for "pem" format if (keyFormat !== undefined && keyFormat !== "pem") { - throw new TypeError('If `keyFormat` is specified, it must be "pem"'); + throw new TypeError( + `If "keyFormat" is specified, it must be "pem": received "${keyFormat}"`, + ); } if (cert !== undefined && key === undefined) { throw new TypeError( - `If \`cert\` is specified, \`key\` must be specified as well for \`${api}\`.`, + `If \`cert\` is specified, \`key\` must be specified as well for \`${api}\``, ); } if (cert === undefined && key !== undefined) { throw new TypeError( - `If \`key\` is specified, \`cert\` must be specified as well for \`${api}\`.`, + `If \`key\` is specified, \`cert\` must be specified as well for \`${api}\``, ); } diff --git a/ext/net/Cargo.toml b/ext/net/Cargo.toml index 02e7a168183609..401ce08ff42b3a 100644 --- a/ext/net/Cargo.toml +++ b/ext/net/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_net" -version = "0.160.0" +version = "0.172.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -17,10 +17,11 @@ path = "lib.rs" deno_core.workspace = true deno_permissions.workspace = true deno_tls.workspace = true +hickory-proto = "0.24" +hickory-resolver.workspace = true pin-project.workspace = true rustls-tokio-stream.workspace = true serde.workspace = true socket2.workspace = true +thiserror.workspace = true tokio.workspace = true -trust-dns-proto = "0.23" -trust-dns-resolver = { version = "0.23", features = ["tokio-runtime", "serde-config"] } diff --git a/ext/net/io.rs b/ext/net/io.rs index f3aed3fcbdc0dd..2907fa398b480a 100644 --- a/ext/net/io.rs +++ b/ext/net/io.rs @@ -1,7 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::generic_error; -use deno_core::error::AnyError; +use deno_core::futures::TryFutureExt; use deno_core::AsyncMutFuture; use deno_core::AsyncRefCell; use deno_core::AsyncResult; @@ -69,25 +68,36 @@ where pub async fn read( self: Rc, data: &mut [u8], - ) -> Result { + ) -> Result { let mut rd = self.rd_borrow_mut().await; let nread = rd.read(data).try_or_cancel(self.cancel_handle()).await?; Ok(nread) } - pub async fn write(self: Rc, data: &[u8]) -> Result { + pub async fn write( + self: Rc, + data: &[u8], + ) -> Result { let mut wr = self.wr_borrow_mut().await; let nwritten = wr.write(data).await?; Ok(nwritten) } - pub async fn shutdown(self: Rc) -> Result<(), AnyError> { + pub async fn shutdown(self: Rc) -> Result<(), std::io::Error> { let mut wr = self.wr_borrow_mut().await; wr.shutdown().await?; Ok(()) } } +#[derive(Debug, thiserror::Error)] +pub enum MapError { + #[error("{0}")] + Io(std::io::Error), + #[error("Unable to get resources")] + NoResources, +} + pub type TcpStreamResource = FullDuplexResource; @@ -100,7 +110,7 @@ impl Resource for TcpStreamResource { } fn shutdown(self: Rc) -> AsyncResult<()> { - Box::pin(self.shutdown()) + Box::pin(self.shutdown().map_err(Into::into)) } fn close(self: Rc) { @@ -109,31 +119,30 @@ impl Resource for TcpStreamResource { } impl TcpStreamResource { - pub fn set_nodelay(self: Rc, nodelay: bool) -> Result<(), AnyError> { - self.map_socket(Box::new(move |socket| Ok(socket.set_nodelay(nodelay)?))) + pub fn set_nodelay(self: Rc, nodelay: bool) -> Result<(), MapError> { + self.map_socket(Box::new(move |socket| socket.set_nodelay(nodelay))) } pub fn set_keepalive( self: Rc, keepalive: bool, - ) -> Result<(), AnyError> { - self - .map_socket(Box::new(move |socket| Ok(socket.set_keepalive(keepalive)?))) + ) -> Result<(), MapError> { + self.map_socket(Box::new(move |socket| socket.set_keepalive(keepalive))) } #[allow(clippy::type_complexity)] fn map_socket( self: Rc, - map: Box Result<(), AnyError>>, - ) -> Result<(), AnyError> { + map: Box Result<(), std::io::Error>>, + ) -> Result<(), MapError> { if let Some(wr) = RcRef::map(self, |r| &r.wr).try_borrow() { let stream = wr.as_ref().as_ref(); let socket = socket2::SockRef::from(stream); - return map(socket); + return map(socket).map_err(MapError::Io); } - Err(generic_error("Unable to get resources")) + Err(MapError::NoResources) } } @@ -153,7 +162,9 @@ impl UnixStreamResource { unreachable!() } #[allow(clippy::unused_async)] - pub async fn shutdown(self: Rc) -> Result<(), AnyError> { + pub async fn shutdown( + self: Rc, + ) -> Result<(), deno_core::error::AnyError> { unreachable!() } pub fn cancel_read_ops(&self) { @@ -170,7 +181,7 @@ impl Resource for UnixStreamResource { } fn shutdown(self: Rc) -> AsyncResult<()> { - Box::pin(self.shutdown()) + Box::pin(self.shutdown().map_err(Into::into)) } fn close(self: Rc) { diff --git a/ext/net/lib.rs b/ext/net/lib.rs index b039965d4c0f82..f482750b38f7cf 100644 --- a/ext/net/lib.rs +++ b/ext/net/lib.rs @@ -7,10 +7,11 @@ pub mod ops_tls; pub mod ops_unix; pub mod raw; pub mod resolve_addr; -mod tcp; +pub mod tcp; use deno_core::error::AnyError; use deno_core::OpState; +use deno_permissions::PermissionCheckError; use deno_tls::rustls::RootCertStore; use deno_tls::RootCertStoreProvider; use std::borrow::Cow; @@ -25,25 +26,25 @@ pub trait NetPermissions { &mut self, host: &(T, Option), api_name: &str, - ) -> Result<(), AnyError>; + ) -> Result<(), PermissionCheckError>; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] fn check_read( &mut self, p: &str, api_name: &str, - ) -> Result; + ) -> Result; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] fn check_write( &mut self, p: &str, api_name: &str, - ) -> Result; + ) -> Result; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] fn check_write_path<'a>( &mut self, p: &'a Path, api_name: &str, - ) -> Result, AnyError>; + ) -> Result, PermissionCheckError>; } impl NetPermissions for deno_permissions::PermissionsContainer { @@ -52,7 +53,7 @@ impl NetPermissions for deno_permissions::PermissionsContainer { &mut self, host: &(T, Option), api_name: &str, - ) -> Result<(), AnyError> { + ) -> Result<(), PermissionCheckError> { deno_permissions::PermissionsContainer::check_net(self, host, api_name) } @@ -61,7 +62,7 @@ impl NetPermissions for deno_permissions::PermissionsContainer { &mut self, path: &str, api_name: &str, - ) -> Result { + ) -> Result { deno_permissions::PermissionsContainer::check_read(self, path, api_name) } @@ -70,7 +71,7 @@ impl NetPermissions for deno_permissions::PermissionsContainer { &mut self, path: &str, api_name: &str, - ) -> Result { + ) -> Result { deno_permissions::PermissionsContainer::check_write(self, path, api_name) } @@ -79,7 +80,7 @@ impl NetPermissions for deno_permissions::PermissionsContainer { &mut self, path: &'a Path, api_name: &str, - ) -> Result, AnyError> { + ) -> Result, PermissionCheckError> { deno_permissions::PermissionsContainer::check_write_path( self, path, api_name, ) diff --git a/ext/net/ops.rs b/ext/net/ops.rs index f2735eda9984e0..8d62bdeb4dcb1e 100644 --- a/ext/net/ops.rs +++ b/ext/net/ops.rs @@ -6,10 +6,6 @@ use crate::resolve_addr::resolve_addr; use crate::resolve_addr::resolve_addr_sync; use crate::tcp::TcpListener; use crate::NetPermissions; -use deno_core::error::bad_resource; -use deno_core::error::custom_error; -use deno_core::error::generic_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::CancelFuture; @@ -22,6 +18,16 @@ use deno_core::OpState; use deno_core::RcRef; use deno_core::Resource; use deno_core::ResourceId; +use hickory_proto::rr::rdata::caa::Value; +use hickory_proto::rr::record_data::RData; +use hickory_proto::rr::record_type::RecordType; +use hickory_resolver::config::NameServerConfigGroup; +use hickory_resolver::config::ResolverConfig; +use hickory_resolver::config::ResolverOpts; +use hickory_resolver::error::ResolveError; +use hickory_resolver::error::ResolveErrorKind; +use hickory_resolver::system_conf; +use hickory_resolver::AsyncResolver; use serde::Deserialize; use serde::Serialize; use socket2::Domain; @@ -37,15 +43,6 @@ use std::rc::Rc; use std::str::FromStr; use tokio::net::TcpStream; use tokio::net::UdpSocket; -use trust_dns_proto::rr::rdata::caa::Value; -use trust_dns_proto::rr::record_data::RData; -use trust_dns_proto::rr::record_type::RecordType; -use trust_dns_resolver::config::NameServerConfigGroup; -use trust_dns_resolver::config::ResolverConfig; -use trust_dns_resolver::config::ResolverOpts; -use trust_dns_resolver::error::ResolveErrorKind; -use trust_dns_resolver::system_conf; -use trust_dns_resolver::AsyncResolver; #[derive(Serialize, Clone, Debug)] #[serde(rename_all = "camelCase")] @@ -68,12 +65,69 @@ impl From for IpAddr { } } -pub(crate) fn accept_err(e: std::io::Error) -> AnyError { - // FIXME(bartlomieju): compatibility with current JS implementation +#[derive(Debug, thiserror::Error)] +pub enum NetError { + #[error("Listener has been closed")] + ListenerClosed, + #[error("Listener already in use")] + ListenerBusy, + #[error("Socket has been closed")] + SocketClosed, + #[error("Socket has been closed")] + SocketClosedNotConnected, + #[error("Socket already in use")] + SocketBusy, + #[error("{0}")] + Io(#[from] std::io::Error), + #[error("Another accept task is ongoing")] + AcceptTaskOngoing, + #[error(transparent)] + Permission(#[from] deno_permissions::PermissionCheckError), + #[error("{0}")] + Resource(deno_core::error::AnyError), + #[error("No resolved address found")] + NoResolvedAddress, + #[error("{0}")] + AddrParse(#[from] std::net::AddrParseError), + #[error("{0}")] + Map(crate::io::MapError), + #[error("{0}")] + Canceled(#[from] deno_core::Canceled), + #[error("{0}")] + DnsNotFound(ResolveError), + #[error("{0}")] + DnsNotConnected(ResolveError), + #[error("{0}")] + DnsTimedOut(ResolveError), + #[error("{0}")] + Dns(#[from] ResolveError), + #[error("Provided record type is not supported")] + UnsupportedRecordType, + #[error("File name or path {0:?} is not valid UTF-8")] + InvalidUtf8(std::ffi::OsString), + #[error("unexpected key type")] + UnexpectedKeyType, + #[error("Invalid hostname: '{0}'")] + InvalidHostname(String), // TypeError + #[error("TCP stream is currently in use")] + TcpStreamBusy, + #[error("{0}")] + Rustls(#[from] deno_tls::rustls::Error), + #[error("{0}")] + Tls(#[from] deno_tls::TlsError), + #[error("Error creating TLS certificate: Deno.listenTls requires a key")] + ListenTlsRequiresKey, // InvalidData + #[error("{0}")] + RootCertStore(deno_core::anyhow::Error), + #[error("{0}")] + Reunite(tokio::net::tcp::ReuniteError), +} + +pub(crate) fn accept_err(e: std::io::Error) -> NetError { if let std::io::ErrorKind::Interrupted = e.kind() { - bad_resource("Listener has been closed") + NetError::ListenerClosed } else { - e.into() + NetError::Io(e) } } @@ -82,15 +136,15 @@ pub(crate) fn accept_err(e: std::io::Error) -> AnyError { pub async fn op_net_accept_tcp( state: Rc>, #[smi] rid: ResourceId, -) -> Result<(ResourceId, IpAddr, IpAddr), AnyError> { +) -> Result<(ResourceId, IpAddr, IpAddr), NetError> { let resource = state .borrow() .resource_table .get::>(rid) - .map_err(|_| bad_resource("Listener has been closed"))?; + .map_err(|_| NetError::ListenerClosed)?; let listener = RcRef::map(&resource, |r| &r.listener) .try_borrow_mut() - .ok_or_else(|| custom_error("Busy", "Another accept task is ongoing"))?; + .ok_or_else(|| NetError::AcceptTaskOngoing)?; let cancel = RcRef::map(resource, |r| &r.cancel); let (tcp_stream, _socket_addr) = listener .accept() @@ -113,12 +167,12 @@ pub async fn op_net_recv_udp( state: Rc>, #[smi] rid: ResourceId, #[buffer] mut buf: JsBuffer, -) -> Result<(usize, IpAddr), AnyError> { +) -> Result<(usize, IpAddr), NetError> { let resource = state .borrow_mut() .resource_table .get::(rid) - .map_err(|_| bad_resource("Socket has been closed"))?; + .map_err(|_| NetError::SocketClosed)?; let socket = RcRef::map(&resource, |r| &r.socket).borrow().await; let cancel_handle = RcRef::map(&resource, |r| &r.cancel); let (nread, remote_addr) = socket @@ -128,14 +182,14 @@ pub async fn op_net_recv_udp( Ok((nread, IpAddr::from(remote_addr))) } -#[op2(async)] +#[op2(async, stack_trace)] #[number] pub async fn op_net_send_udp( state: Rc>, #[smi] rid: ResourceId, #[serde] addr: IpAddr, #[buffer] zero_copy: JsBuffer, -) -> Result +) -> Result where NP: NetPermissions + 'static, { @@ -149,13 +203,13 @@ where let addr = resolve_addr(&addr.hostname, addr.port) .await? .next() - .ok_or_else(|| generic_error("No resolved address found"))?; + .ok_or(NetError::NoResolvedAddress)?; let resource = state .borrow_mut() .resource_table .get::(rid) - .map_err(|_| bad_resource("Socket has been closed"))?; + .map_err(|_| NetError::SocketClosed)?; let socket = RcRef::map(&resource, |r| &r.socket).borrow().await; let nwritten = socket.send_to(&zero_copy, &addr).await?; @@ -168,12 +222,12 @@ pub async fn op_net_join_multi_v4_udp( #[smi] rid: ResourceId, #[string] address: String, #[string] multi_interface: String, -) -> Result<(), AnyError> { +) -> Result<(), NetError> { let resource = state .borrow_mut() .resource_table .get::(rid) - .map_err(|_| bad_resource("Socket has been closed"))?; + .map_err(|_| NetError::SocketClosed)?; let socket = RcRef::map(&resource, |r| &r.socket).borrow().await; let addr = Ipv4Addr::from_str(address.as_str())?; @@ -190,12 +244,12 @@ pub async fn op_net_join_multi_v6_udp( #[smi] rid: ResourceId, #[string] address: String, #[smi] multi_interface: u32, -) -> Result<(), AnyError> { +) -> Result<(), NetError> { let resource = state .borrow_mut() .resource_table .get::(rid) - .map_err(|_| bad_resource("Socket has been closed"))?; + .map_err(|_| NetError::SocketClosed)?; let socket = RcRef::map(&resource, |r| &r.socket).borrow().await; let addr = Ipv6Addr::from_str(address.as_str())?; @@ -211,12 +265,12 @@ pub async fn op_net_leave_multi_v4_udp( #[smi] rid: ResourceId, #[string] address: String, #[string] multi_interface: String, -) -> Result<(), AnyError> { +) -> Result<(), NetError> { let resource = state .borrow_mut() .resource_table .get::(rid) - .map_err(|_| bad_resource("Socket has been closed"))?; + .map_err(|_| NetError::SocketClosed)?; let socket = RcRef::map(&resource, |r| &r.socket).borrow().await; let addr = Ipv4Addr::from_str(address.as_str())?; @@ -233,12 +287,12 @@ pub async fn op_net_leave_multi_v6_udp( #[smi] rid: ResourceId, #[string] address: String, #[smi] multi_interface: u32, -) -> Result<(), AnyError> { +) -> Result<(), NetError> { let resource = state .borrow_mut() .resource_table .get::(rid) - .map_err(|_| bad_resource("Socket has been closed"))?; + .map_err(|_| NetError::SocketClosed)?; let socket = RcRef::map(&resource, |r| &r.socket).borrow().await; let addr = Ipv6Addr::from_str(address.as_str())?; @@ -254,16 +308,16 @@ pub async fn op_net_set_multi_loopback_udp( #[smi] rid: ResourceId, is_v4_membership: bool, loopback: bool, -) -> Result<(), AnyError> { +) -> Result<(), NetError> { let resource = state .borrow_mut() .resource_table .get::(rid) - .map_err(|_| bad_resource("Socket has been closed"))?; + .map_err(|_| NetError::SocketClosed)?; let socket = RcRef::map(&resource, |r| &r.socket).borrow().await; if is_v4_membership { - socket.set_multicast_loop_v4(loopback)? + socket.set_multicast_loop_v4(loopback)?; } else { socket.set_multicast_loop_v6(loopback)?; } @@ -276,12 +330,12 @@ pub async fn op_net_set_multi_ttl_udp( state: Rc>, #[smi] rid: ResourceId, #[smi] ttl: u32, -) -> Result<(), AnyError> { +) -> Result<(), NetError> { let resource = state .borrow_mut() .resource_table .get::(rid) - .map_err(|_| bad_resource("Socket has been closed"))?; + .map_err(|_| NetError::SocketClosed)?; let socket = RcRef::map(&resource, |r| &r.socket).borrow().await; socket.set_multicast_ttl_v4(ttl)?; @@ -289,12 +343,12 @@ pub async fn op_net_set_multi_ttl_udp( Ok(()) } -#[op2(async)] +#[op2(async, stack_trace)] #[serde] pub async fn op_net_connect_tcp( state: Rc>, #[serde] addr: IpAddr, -) -> Result<(ResourceId, IpAddr, IpAddr), AnyError> +) -> Result<(ResourceId, IpAddr, IpAddr), NetError> where NP: NetPermissions + 'static, { @@ -305,7 +359,7 @@ where pub async fn op_net_connect_tcp_inner( state: Rc>, addr: IpAddr, -) -> Result<(ResourceId, IpAddr, IpAddr), AnyError> +) -> Result<(ResourceId, IpAddr, IpAddr), NetError> where NP: NetPermissions + 'static, { @@ -319,7 +373,7 @@ where let addr = resolve_addr(&addr.hostname, addr.port) .await? .next() - .ok_or_else(|| generic_error("No resolved address found"))?; + .ok_or_else(|| NetError::NoResolvedAddress)?; let tcp_stream = TcpStream::connect(&addr).await?; let local_addr = tcp_stream.local_addr()?; let remote_addr = tcp_stream.peer_addr()?; @@ -347,14 +401,14 @@ impl Resource for UdpSocketResource { } } -#[op2] +#[op2(stack_trace)] #[serde] pub fn op_net_listen_tcp( state: &mut OpState, #[serde] addr: IpAddr, reuse_port: bool, load_balanced: bool, -) -> Result<(ResourceId, IpAddr), AnyError> +) -> Result<(ResourceId, IpAddr), NetError> where NP: NetPermissions + 'static, { @@ -366,7 +420,7 @@ where .check_net(&(&addr.hostname, Some(addr.port)), "Deno.listen()")?; let addr = resolve_addr_sync(&addr.hostname, addr.port)? .next() - .ok_or_else(|| generic_error("No resolved address found"))?; + .ok_or_else(|| NetError::NoResolvedAddress)?; let listener = if load_balanced { TcpListener::bind_load_balanced(addr) @@ -385,7 +439,7 @@ fn net_listen_udp( addr: IpAddr, reuse_address: bool, loopback: bool, -) -> Result<(ResourceId, IpAddr), AnyError> +) -> Result<(ResourceId, IpAddr), NetError> where NP: NetPermissions + 'static, { @@ -394,7 +448,7 @@ where .check_net(&(&addr.hostname, Some(addr.port)), "Deno.listenDatagram()")?; let addr = resolve_addr_sync(&addr.hostname, addr.port)? .next() - .ok_or_else(|| generic_error("No resolved address found"))?; + .ok_or_else(|| NetError::NoResolvedAddress)?; let domain = if addr.is_ipv4() { Domain::IPV4 @@ -447,14 +501,14 @@ where Ok((rid, IpAddr::from(local_addr))) } -#[op2] +#[op2(stack_trace)] #[serde] pub fn op_net_listen_udp( state: &mut OpState, #[serde] addr: IpAddr, reuse_address: bool, loopback: bool, -) -> Result<(ResourceId, IpAddr), AnyError> +) -> Result<(ResourceId, IpAddr), NetError> where NP: NetPermissions + 'static, { @@ -462,14 +516,14 @@ where net_listen_udp::(state, addr, reuse_address, loopback) } -#[op2] +#[op2(stack_trace)] #[serde] pub fn op_node_unstable_net_listen_udp( state: &mut OpState, #[serde] addr: IpAddr, reuse_address: bool, loopback: bool, -) -> Result<(ResourceId, IpAddr), AnyError> +) -> Result<(ResourceId, IpAddr), NetError> where NP: NetPermissions + 'static, { @@ -547,12 +601,12 @@ pub struct NameServer { port: u16, } -#[op2(async)] +#[op2(async, stack_trace)] #[serde] pub async fn op_dns_resolve( state: Rc>, #[serde] args: ResolveAddrArgs, -) -> Result, AnyError> +) -> Result, NetError> where NP: NetPermissions + 'static, { @@ -619,22 +673,17 @@ where }; lookup - .map_err(|e| { - let message = format!("{e}"); - match e.kind() { - ResolveErrorKind::NoRecordsFound { .. } => { - custom_error("NotFound", message) - } - ResolveErrorKind::Message("No connections available") => { - custom_error("NotConnected", message) - } - ResolveErrorKind::Timeout => custom_error("TimedOut", message), - _ => generic_error(message), + .map_err(|e| match e.kind() { + ResolveErrorKind::NoRecordsFound { .. } => NetError::DnsNotFound(e), + ResolveErrorKind::Message("No connections available") => { + NetError::DnsNotConnected(e) } + ResolveErrorKind::Timeout => NetError::DnsTimedOut(e), + _ => NetError::Dns(e), })? .iter() .filter_map(|rdata| rdata_to_return_record(record_type)(rdata).transpose()) - .collect::, AnyError>>() + .collect::, NetError>>() } #[op2(fast)] @@ -642,7 +691,7 @@ pub fn op_set_nodelay( state: &mut OpState, #[smi] rid: ResourceId, nodelay: bool, -) -> Result<(), AnyError> { +) -> Result<(), NetError> { op_set_nodelay_inner(state, rid, nodelay) } @@ -651,10 +700,12 @@ pub fn op_set_nodelay_inner( state: &mut OpState, rid: ResourceId, nodelay: bool, -) -> Result<(), AnyError> { - let resource: Rc = - state.resource_table.get::(rid)?; - resource.set_nodelay(nodelay) +) -> Result<(), NetError> { + let resource: Rc = state + .resource_table + .get::(rid) + .map_err(NetError::Resource)?; + resource.set_nodelay(nodelay).map_err(NetError::Map) } #[op2(fast)] @@ -662,7 +713,7 @@ pub fn op_set_keepalive( state: &mut OpState, #[smi] rid: ResourceId, keepalive: bool, -) -> Result<(), AnyError> { +) -> Result<(), NetError> { op_set_keepalive_inner(state, rid, keepalive) } @@ -671,17 +722,19 @@ pub fn op_set_keepalive_inner( state: &mut OpState, rid: ResourceId, keepalive: bool, -) -> Result<(), AnyError> { - let resource: Rc = - state.resource_table.get::(rid)?; - resource.set_keepalive(keepalive) +) -> Result<(), NetError> { + let resource: Rc = state + .resource_table + .get::(rid) + .map_err(NetError::Resource)?; + resource.set_keepalive(keepalive).map_err(NetError::Map) } fn rdata_to_return_record( ty: RecordType, -) -> impl Fn(&RData) -> Result, AnyError> { +) -> impl Fn(&RData) -> Result, NetError> { use RecordType::*; - move |r: &RData| -> Result, AnyError> { + move |r: &RData| -> Result, NetError> { let record = match ty { A => r.as_a().map(ToString::to_string).map(DnsReturnRecord::A), AAAA => r @@ -762,12 +815,7 @@ fn rdata_to_return_record( .collect(); DnsReturnRecord::Txt(texts) }), - _ => { - return Err(custom_error( - "NotSupported", - "Provided record type is not supported", - )) - } + _ => return Err(NetError::UnsupportedRecordType), }; Ok(record) } @@ -779,6 +827,22 @@ mod tests { use deno_core::futures::FutureExt; use deno_core::JsRuntime; use deno_core::RuntimeOptions; + use deno_permissions::PermissionCheckError; + use hickory_proto::rr::rdata::a::A; + use hickory_proto::rr::rdata::aaaa::AAAA; + use hickory_proto::rr::rdata::caa::KeyValue; + use hickory_proto::rr::rdata::caa::CAA; + use hickory_proto::rr::rdata::mx::MX; + use hickory_proto::rr::rdata::name::ANAME; + use hickory_proto::rr::rdata::name::CNAME; + use hickory_proto::rr::rdata::name::NS; + use hickory_proto::rr::rdata::name::PTR; + use hickory_proto::rr::rdata::naptr::NAPTR; + use hickory_proto::rr::rdata::srv::SRV; + use hickory_proto::rr::rdata::txt::TXT; + use hickory_proto::rr::rdata::SOA; + use hickory_proto::rr::record_data::RData; + use hickory_proto::rr::Name; use socket2::SockRef; use std::net::Ipv4Addr; use std::net::Ipv6Addr; @@ -787,21 +851,6 @@ mod tests { use std::path::PathBuf; use std::sync::Arc; use std::sync::Mutex; - use trust_dns_proto::rr::rdata::a::A; - use trust_dns_proto::rr::rdata::aaaa::AAAA; - use trust_dns_proto::rr::rdata::caa::KeyValue; - use trust_dns_proto::rr::rdata::caa::CAA; - use trust_dns_proto::rr::rdata::mx::MX; - use trust_dns_proto::rr::rdata::name::ANAME; - use trust_dns_proto::rr::rdata::name::CNAME; - use trust_dns_proto::rr::rdata::name::NS; - use trust_dns_proto::rr::rdata::name::PTR; - use trust_dns_proto::rr::rdata::naptr::NAPTR; - use trust_dns_proto::rr::rdata::srv::SRV; - use trust_dns_proto::rr::rdata::txt::TXT; - use trust_dns_proto::rr::rdata::SOA; - use trust_dns_proto::rr::record_data::RData; - use trust_dns_proto::rr::Name; #[test] fn rdata_to_return_record_a() { @@ -986,7 +1035,7 @@ mod tests { &mut self, _host: &(T, Option), _api_name: &str, - ) -> Result<(), AnyError> { + ) -> Result<(), PermissionCheckError> { Ok(()) } @@ -994,7 +1043,7 @@ mod tests { &mut self, p: &str, _api_name: &str, - ) -> Result { + ) -> Result { Ok(PathBuf::from(p)) } @@ -1002,7 +1051,7 @@ mod tests { &mut self, p: &str, _api_name: &str, - ) -> Result { + ) -> Result { Ok(PathBuf::from(p)) } @@ -1010,7 +1059,7 @@ mod tests { &mut self, p: &'a Path, _api_name: &str, - ) -> Result, AnyError> { + ) -> Result, PermissionCheckError> { Ok(Cow::Borrowed(p)) } } @@ -1092,7 +1141,7 @@ mod tests { let vals = result.unwrap(); rid = rid.or(Some(vals.0)); } - }; + } let rid = rid.unwrap(); let state = runtime.op_state(); diff --git a/ext/net/ops_tls.rs b/ext/net/ops_tls.rs index 064da5818cccf1..4d2073fd092c92 100644 --- a/ext/net/ops_tls.rs +++ b/ext/net/ops_tls.rs @@ -2,6 +2,7 @@ use crate::io::TcpStreamResource; use crate::ops::IpAddr; +use crate::ops::NetError; use crate::ops::TlsHandshakeInfo; use crate::raw::NetworkListenerResource; use crate::resolve_addr::resolve_addr; @@ -10,13 +11,7 @@ use crate::tcp::TcpListener; use crate::DefaultTlsOptions; use crate::NetPermissions; use crate::UnsafelyIgnoreCertificateErrors; -use deno_core::anyhow::anyhow; -use deno_core::anyhow::bail; -use deno_core::error::bad_resource; -use deno_core::error::custom_error; -use deno_core::error::generic_error; -use deno_core::error::invalid_hostname; -use deno_core::error::AnyError; +use deno_core::futures::TryFutureExt; use deno_core::op2; use deno_core::v8; use deno_core::AsyncRefCell; @@ -118,20 +113,23 @@ impl TlsStreamResource { pub async fn read( self: Rc, data: &mut [u8], - ) -> Result { + ) -> Result { let mut rd = RcRef::map(&self, |r| &r.rd).borrow_mut().await; let cancel_handle = RcRef::map(&self, |r| &r.cancel_handle); - Ok(rd.read(data).try_or_cancel(cancel_handle).await?) + rd.read(data).try_or_cancel(cancel_handle).await } - pub async fn write(self: Rc, data: &[u8]) -> Result { + pub async fn write( + self: Rc, + data: &[u8], + ) -> Result { let mut wr = RcRef::map(self, |r| &r.wr).borrow_mut().await; let nwritten = wr.write(data).await?; wr.flush().await?; Ok(nwritten) } - pub async fn shutdown(self: Rc) -> Result<(), AnyError> { + pub async fn shutdown(self: Rc) -> Result<(), std::io::Error> { let mut wr = RcRef::map(self, |r| &r.wr).borrow_mut().await; wr.shutdown().await?; Ok(()) @@ -139,7 +137,7 @@ impl TlsStreamResource { pub async fn handshake( self: &Rc, - ) -> Result { + ) -> Result { if let Some(tls_info) = &*self.handshake_info.borrow() { return Ok(tls_info.clone()); } @@ -164,7 +162,7 @@ impl Resource for TlsStreamResource { } fn shutdown(self: Rc) -> AsyncResult<()> { - Box::pin(self.shutdown()) + Box::pin(self.shutdown().map_err(Into::into)) } fn close(self: Rc) { @@ -201,7 +199,7 @@ pub fn op_tls_key_null() -> TlsKeysHolder { pub fn op_tls_key_static( #[string] cert: &str, #[string] key: &str, -) -> Result { +) -> Result { let cert = load_certs(&mut BufReader::new(cert.as_bytes()))?; let key = load_private_keys(key.as_bytes())? .into_iter() @@ -236,9 +234,9 @@ pub fn op_tls_cert_resolver_resolve( #[cppgc] lookup: &TlsKeyLookup, #[string] sni: String, #[cppgc] key: &TlsKeysHolder, -) -> Result<(), AnyError> { +) -> Result<(), NetError> { let TlsKeys::Static(key) = key.take() else { - bail!("unexpected key type"); + return Err(NetError::UnexpectedKeyType); }; lookup.resolve(sni, Ok(key)); Ok(()) @@ -250,15 +248,15 @@ pub fn op_tls_cert_resolver_resolve_error( #[string] sni: String, #[string] error: String, ) { - lookup.resolve(sni, Err(anyhow!(error))) + lookup.resolve(sni, Err(error)) } -#[op2] +#[op2(stack_trace)] #[serde] pub fn op_tls_start( state: Rc>, #[serde] args: StartTlsArgs, -) -> Result<(ResourceId, IpAddr, IpAddr), AnyError> +) -> Result<(ResourceId, IpAddr, IpAddr), NetError> where NP: NetPermissions + 'static, { @@ -271,7 +269,9 @@ where { let mut s = state.borrow_mut(); let permissions = s.borrow_mut::(); - permissions.check_net(&(&hostname, Some(0)), "Deno.startTls()")?; + permissions + .check_net(&(&hostname, Some(0)), "Deno.startTls()") + .map_err(NetError::Permission)?; } let ca_certs = args @@ -281,7 +281,7 @@ where .collect::>(); let hostname_dns = ServerName::try_from(hostname.to_string()) - .map_err(|_| invalid_hostname(&hostname))?; + .map_err(|_| NetError::InvalidHostname(hostname))?; let unsafely_ignore_certificate_errors = state .borrow() @@ -291,19 +291,21 @@ where let root_cert_store = state .borrow() .borrow::() - .root_cert_store()?; + .root_cert_store() + .map_err(NetError::RootCertStore)?; let resource_rc = state .borrow_mut() .resource_table - .take::(rid)?; + .take::(rid) + .map_err(NetError::Resource)?; // This TCP connection might be used somewhere else. If it's the case, we cannot proceed with the - // process of starting a TLS connection on top of this TCP connection, so we just return a bad - // resource error. See also: https://github.com/denoland/deno/pull/16242 - let resource = Rc::try_unwrap(resource_rc) - .map_err(|_| bad_resource("TCP stream is currently in use"))?; + // process of starting a TLS connection on top of this TCP connection, so we just return a Busy error. + // See also: https://github.com/denoland/deno/pull/16242 + let resource = + Rc::try_unwrap(resource_rc).map_err(|_| NetError::TcpStreamBusy)?; let (read_half, write_half) = resource.into_inner(); - let tcp_stream = read_half.reunite(write_half)?; + let tcp_stream = read_half.reunite(write_half).map_err(NetError::Reunite)?; let local_addr = tcp_stream.local_addr()?; let remote_addr = tcp_stream.peer_addr()?; @@ -338,14 +340,14 @@ where Ok((rid, IpAddr::from(local_addr), IpAddr::from(remote_addr))) } -#[op2(async)] +#[op2(async, stack_trace)] #[serde] pub async fn op_net_connect_tls( state: Rc>, #[serde] addr: IpAddr, #[serde] args: ConnectTlsArgs, #[cppgc] key_pair: &TlsKeysHolder, -) -> Result<(ResourceId, IpAddr, IpAddr), AnyError> +) -> Result<(ResourceId, IpAddr, IpAddr), NetError> where NP: NetPermissions + 'static, { @@ -359,9 +361,14 @@ where let mut s = state.borrow_mut(); let permissions = s.borrow_mut::(); permissions - .check_net(&(&addr.hostname, Some(addr.port)), "Deno.connectTls()")?; + .check_net(&(&addr.hostname, Some(addr.port)), "Deno.connectTls()") + .map_err(NetError::Permission)?; if let Some(path) = cert_file { - Some(permissions.check_read(path, "Deno.connectTls()")?) + Some( + permissions + .check_read(path, "Deno.connectTls()") + .map_err(NetError::Permission)?, + ) } else { None } @@ -382,17 +389,18 @@ where let root_cert_store = state .borrow() .borrow::() - .root_cert_store()?; + .root_cert_store() + .map_err(NetError::RootCertStore)?; let hostname_dns = if let Some(server_name) = args.server_name { ServerName::try_from(server_name) } else { ServerName::try_from(addr.hostname.clone()) } - .map_err(|_| invalid_hostname(&addr.hostname))?; + .map_err(|_| NetError::InvalidHostname(addr.hostname.clone()))?; let connect_addr = resolve_addr(&addr.hostname, addr.port) .await? .next() - .ok_or_else(|| generic_error("No resolved address found"))?; + .ok_or_else(|| NetError::NoResolvedAddress)?; let tcp_stream = TcpStream::connect(connect_addr).await?; let local_addr = tcp_stream.local_addr()?; let remote_addr = tcp_stream.peer_addr()?; @@ -437,14 +445,14 @@ pub struct ListenTlsArgs { load_balanced: bool, } -#[op2] +#[op2(stack_trace)] #[serde] pub fn op_net_listen_tls( state: &mut OpState, #[serde] addr: IpAddr, #[serde] args: ListenTlsArgs, #[cppgc] keys: &TlsKeysHolder, -) -> Result<(ResourceId, IpAddr), AnyError> +) -> Result<(ResourceId, IpAddr), NetError> where NP: NetPermissions + 'static, { @@ -455,12 +463,13 @@ where { let permissions = state.borrow_mut::(); permissions - .check_net(&(&addr.hostname, Some(addr.port)), "Deno.listenTls()")?; + .check_net(&(&addr.hostname, Some(addr.port)), "Deno.listenTls()") + .map_err(NetError::Permission)?; } let bind_addr = resolve_addr_sync(&addr.hostname, addr.port)? .next() - .ok_or_else(|| generic_error("No resolved address found"))?; + .ok_or(NetError::NoResolvedAddress)?; let tcp_listener = if args.load_balanced { TcpListener::bind_load_balanced(bind_addr) @@ -475,28 +484,24 @@ where .map(|s| s.into_bytes()) .collect(); let listener = match keys.take() { - TlsKeys::Null => Err(anyhow!("Deno.listenTls requires a key")), + TlsKeys::Null => return Err(NetError::ListenTlsRequiresKey), TlsKeys::Static(TlsKey(cert, key)) => { let mut tls_config = ServerConfig::builder() .with_no_client_auth() - .with_single_cert(cert, key) - .map_err(|e| anyhow!(e))?; + .with_single_cert(cert, key)?; tls_config.alpn_protocols = alpn; - Ok(TlsListener { + TlsListener { tcp_listener, tls_config: Some(tls_config.into()), server_config_provider: None, - }) + } } - TlsKeys::Resolver(resolver) => Ok(TlsListener { + TlsKeys::Resolver(resolver) => TlsListener { tcp_listener, tls_config: None, server_config_provider: Some(resolver.into_server_config_provider(alpn)), - }), - } - .map_err(|e| { - custom_error("InvalidData", "Error creating TLS certificate").context(e) - })?; + }, + }; let tls_listener_resource = NetworkListenerResource::new(listener); @@ -510,24 +515,23 @@ where pub async fn op_net_accept_tls( state: Rc>, #[smi] rid: ResourceId, -) -> Result<(ResourceId, IpAddr, IpAddr), AnyError> { +) -> Result<(ResourceId, IpAddr, IpAddr), NetError> { let resource = state .borrow() .resource_table .get::>(rid) - .map_err(|_| bad_resource("Listener has been closed"))?; + .map_err(|_| NetError::ListenerClosed)?; let cancel_handle = RcRef::map(&resource, |r| &r.cancel); let listener = RcRef::map(&resource, |r| &r.listener) .try_borrow_mut() - .ok_or_else(|| custom_error("Busy", "Another accept task is ongoing"))?; + .ok_or_else(|| NetError::AcceptTaskOngoing)?; let (tls_stream, remote_addr) = match listener.accept().try_or_cancel(&cancel_handle).await { Ok(tuple) => tuple, Err(err) if err.kind() == ErrorKind::Interrupted => { - // FIXME(bartlomieju): compatibility with current JS implementation. - return Err(bad_resource("Listener has been closed")); + return Err(NetError::ListenerClosed); } Err(err) => return Err(err.into()), }; @@ -548,11 +552,11 @@ pub async fn op_net_accept_tls( pub async fn op_tls_handshake( state: Rc>, #[smi] rid: ResourceId, -) -> Result { +) -> Result { let resource = state .borrow() .resource_table .get::(rid) - .map_err(|_| bad_resource("Listener has been closed"))?; - resource.handshake().await + .map_err(|_| NetError::ListenerClosed)?; + resource.handshake().await.map_err(Into::into) } diff --git a/ext/net/ops_unix.rs b/ext/net/ops_unix.rs index 95293284f07b1c..483dc99b400c58 100644 --- a/ext/net/ops_unix.rs +++ b/ext/net/ops_unix.rs @@ -1,11 +1,9 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use crate::io::UnixStreamResource; +use crate::ops::NetError; use crate::raw::NetworkListenerResource; use crate::NetPermissions; -use deno_core::error::bad_resource; -use deno_core::error::custom_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::AsyncRefCell; use deno_core::CancelHandle; @@ -26,11 +24,8 @@ use tokio::net::UnixListener; pub use tokio::net::UnixStream; /// A utility function to map OsStrings to Strings -pub fn into_string(s: std::ffi::OsString) -> Result { - s.into_string().map_err(|s| { - let message = format!("File name or path {s:?} is not valid UTF-8"); - custom_error("InvalidData", message) - }) +pub fn into_string(s: std::ffi::OsString) -> Result { + s.into_string().map_err(NetError::InvalidUtf8) } pub struct UnixDatagramResource { @@ -63,15 +58,15 @@ pub struct UnixListenArgs { pub async fn op_net_accept_unix( state: Rc>, #[smi] rid: ResourceId, -) -> Result<(ResourceId, Option, Option), AnyError> { +) -> Result<(ResourceId, Option, Option), NetError> { let resource = state .borrow() .resource_table .get::>(rid) - .map_err(|_| bad_resource("Listener has been closed"))?; + .map_err(|_| NetError::ListenerClosed)?; let listener = RcRef::map(&resource, |r| &r.listener) .try_borrow_mut() - .ok_or_else(|| custom_error("Busy", "Listener already in use"))?; + .ok_or(NetError::ListenerBusy)?; let cancel = RcRef::map(resource, |r| &r.cancel); let (unix_stream, _socket_addr) = listener .accept() @@ -90,12 +85,12 @@ pub async fn op_net_accept_unix( Ok((rid, local_addr_path, remote_addr_path)) } -#[op2(async)] +#[op2(async, stack_trace)] #[serde] pub async fn op_net_connect_unix( state: Rc>, #[string] address_path: String, -) -> Result<(ResourceId, Option, Option), AnyError> +) -> Result<(ResourceId, Option, Option), NetError> where NP: NetPermissions + 'static, { @@ -103,10 +98,12 @@ where let mut state_ = state.borrow_mut(); let address_path = state_ .borrow_mut::() - .check_read(&address_path, "Deno.connect()")?; + .check_read(&address_path, "Deno.connect()") + .map_err(NetError::Permission)?; _ = state_ .borrow_mut::() - .check_write_path(&address_path, "Deno.connect()")?; + .check_write_path(&address_path, "Deno.connect()") + .map_err(NetError::Permission)?; address_path }; let unix_stream = UnixStream::connect(&address_path).await?; @@ -121,21 +118,21 @@ where Ok((rid, local_addr_path, remote_addr_path)) } -#[op2(async)] +#[op2(async, stack_trace)] #[serde] pub async fn op_net_recv_unixpacket( state: Rc>, #[smi] rid: ResourceId, #[buffer] mut buf: JsBuffer, -) -> Result<(usize, Option), AnyError> { +) -> Result<(usize, Option), NetError> { let resource = state .borrow() .resource_table .get::(rid) - .map_err(|_| bad_resource("Socket has been closed"))?; + .map_err(|_| NetError::SocketClosed)?; let socket = RcRef::map(&resource, |r| &r.socket) .try_borrow_mut() - .ok_or_else(|| custom_error("Busy", "Socket already in use"))?; + .ok_or(NetError::SocketBusy)?; let cancel = RcRef::map(resource, |r| &r.cancel); let (nread, remote_addr) = socket.recv_from(&mut buf).try_or_cancel(cancel).await?; @@ -143,50 +140,55 @@ pub async fn op_net_recv_unixpacket( Ok((nread, path)) } -#[op2(async)] +#[op2(async, stack_trace)] #[number] pub async fn op_net_send_unixpacket( state: Rc>, #[smi] rid: ResourceId, #[string] address_path: String, #[buffer] zero_copy: JsBuffer, -) -> Result +) -> Result where NP: NetPermissions + 'static, { let address_path = { let mut s = state.borrow_mut(); s.borrow_mut::() - .check_write(&address_path, "Deno.DatagramConn.send()")? + .check_write(&address_path, "Deno.DatagramConn.send()") + .map_err(NetError::Permission)? }; let resource = state .borrow() .resource_table .get::(rid) - .map_err(|_| custom_error("NotConnected", "Socket has been closed"))?; + .map_err(|_| NetError::SocketClosedNotConnected)?; let socket = RcRef::map(&resource, |r| &r.socket) .try_borrow_mut() - .ok_or_else(|| custom_error("Busy", "Socket already in use"))?; + .ok_or(NetError::SocketBusy)?; let nwritten = socket.send_to(&zero_copy, address_path).await?; Ok(nwritten) } -#[op2] +#[op2(stack_trace)] #[serde] pub fn op_net_listen_unix( state: &mut OpState, #[string] address_path: String, #[string] api_name: String, -) -> Result<(ResourceId, Option), AnyError> +) -> Result<(ResourceId, Option), NetError> where NP: NetPermissions + 'static, { let permissions = state.borrow_mut::(); let api_call_expr = format!("{}()", api_name); - let address_path = permissions.check_read(&address_path, &api_call_expr)?; - _ = permissions.check_write_path(&address_path, &api_call_expr)?; + let address_path = permissions + .check_read(&address_path, &api_call_expr) + .map_err(NetError::Permission)?; + _ = permissions + .check_write_path(&address_path, &api_call_expr) + .map_err(NetError::Permission)?; let listener = UnixListener::bind(address_path)?; let local_addr = listener.local_addr()?; let pathname = local_addr.as_pathname().map(pathstring).transpose()?; @@ -198,14 +200,17 @@ where pub fn net_listen_unixpacket( state: &mut OpState, address_path: String, -) -> Result<(ResourceId, Option), AnyError> +) -> Result<(ResourceId, Option), NetError> where NP: NetPermissions + 'static, { let permissions = state.borrow_mut::(); - let address_path = - permissions.check_read(&address_path, "Deno.listenDatagram()")?; - _ = permissions.check_write_path(&address_path, "Deno.listenDatagram()")?; + let address_path = permissions + .check_read(&address_path, "Deno.listenDatagram()") + .map_err(NetError::Permission)?; + _ = permissions + .check_write_path(&address_path, "Deno.listenDatagram()") + .map_err(NetError::Permission)?; let socket = UnixDatagram::bind(address_path)?; let local_addr = socket.local_addr()?; let pathname = local_addr.as_pathname().map(pathstring).transpose()?; @@ -217,12 +222,12 @@ where Ok((rid, pathname)) } -#[op2] +#[op2(stack_trace)] #[serde] pub fn op_net_listen_unixpacket( state: &mut OpState, #[string] path: String, -) -> Result<(ResourceId, Option), AnyError> +) -> Result<(ResourceId, Option), NetError> where NP: NetPermissions + 'static, { @@ -230,18 +235,18 @@ where net_listen_unixpacket::(state, path) } -#[op2] +#[op2(stack_trace)] #[serde] pub fn op_node_unstable_net_listen_unixpacket( state: &mut OpState, #[string] path: String, -) -> Result<(ResourceId, Option), AnyError> +) -> Result<(ResourceId, Option), NetError> where NP: NetPermissions + 'static, { net_listen_unixpacket::(state, path) } -pub fn pathstring(pathname: &Path) -> Result { +pub fn pathstring(pathname: &Path) -> Result { into_string(pathname.into()) } diff --git a/ext/net/raw.rs b/ext/net/raw.rs index f2de760652aa18..a2ebfb5acb5712 100644 --- a/ext/net/raw.rs +++ b/ext/net/raw.rs @@ -1,8 +1,8 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use crate::io::TcpStreamResource; use crate::ops_tls::TlsStreamResource; -use deno_core::error::bad_resource; use deno_core::error::bad_resource_id; +use deno_core::error::custom_error; use deno_core::error::AnyError; use deno_core::AsyncRefCell; use deno_core::CancelHandle; @@ -70,7 +70,7 @@ impl NetworkListenerResource { ) -> Result, AnyError> { if let Ok(resource_rc) = resource_table.take::(listener_rid) { let resource = Rc::try_unwrap(resource_rc) - .map_err(|_| bad_resource("Listener is currently in use"))?; + .map_err(|_| custom_error("Busy", "Listener is currently in use"))?; return Ok(Some(resource.listener.into_inner().into())); } Ok(None) @@ -334,7 +334,7 @@ pub fn take_network_stream_resource( { // This TCP connection might be used somewhere else. let resource = Rc::try_unwrap(resource_rc) - .map_err(|_| bad_resource("TCP stream is currently in use"))?; + .map_err(|_| custom_error("Busy", "TCP stream is currently in use"))?; let (read_half, write_half) = resource.into_inner(); let tcp_stream = read_half.reunite(write_half)?; return Ok(NetworkStream::Tcp(tcp_stream)); @@ -344,7 +344,7 @@ pub fn take_network_stream_resource( { // This TLS connection might be used somewhere else. let resource = Rc::try_unwrap(resource_rc) - .map_err(|_| bad_resource("TLS stream is currently in use"))?; + .map_err(|_| custom_error("Busy", "TLS stream is currently in use"))?; let (read_half, write_half) = resource.into_inner(); let tls_stream = read_half.unsplit(write_half); return Ok(NetworkStream::Tls(tls_stream)); @@ -356,7 +356,7 @@ pub fn take_network_stream_resource( { // This UNIX socket might be used somewhere else. let resource = Rc::try_unwrap(resource_rc) - .map_err(|_| bad_resource("UNIX stream is currently in use"))?; + .map_err(|_| custom_error("Busy", "Unix socket is currently in use"))?; let (read_half, write_half) = resource.into_inner(); let unix_stream = read_half.reunite(write_half)?; return Ok(NetworkStream::Unix(unix_stream)); diff --git a/ext/net/resolve_addr.rs b/ext/net/resolve_addr.rs index 8bbdd5192cbc60..3a97081eac2ea6 100644 --- a/ext/net/resolve_addr.rs +++ b/ext/net/resolve_addr.rs @@ -1,6 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::AnyError; use std::net::SocketAddr; use std::net::ToSocketAddrs; use tokio::net::lookup_host; @@ -9,7 +8,7 @@ use tokio::net::lookup_host; pub async fn resolve_addr( hostname: &str, port: u16, -) -> Result + '_, AnyError> { +) -> Result + '_, std::io::Error> { let addr_port_pair = make_addr_port_pair(hostname, port); let result = lookup_host(addr_port_pair).await?; Ok(result) @@ -19,7 +18,7 @@ pub async fn resolve_addr( pub fn resolve_addr_sync( hostname: &str, port: u16, -) -> Result, AnyError> { +) -> Result, std::io::Error> { let addr_port_pair = make_addr_port_pair(hostname, port); let result = addr_port_pair.to_socket_addrs()?; Ok(result) diff --git a/ext/node/Cargo.toml b/ext/node/Cargo.toml index 1cd97e3e81046a..89def9b0bc1da5 100644 --- a/ext/node/Cargo.toml +++ b/ext/node/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_node" -version = "0.105.0" +version = "0.117.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -22,6 +22,7 @@ aes.workspace = true async-trait.workspace = true base64.workspace = true blake2 = "0.10.6" +boxed_error.workspace = true brotli.workspace = true bytes.workspace = true cbc.workspace = true @@ -34,6 +35,7 @@ deno_io.workspace = true deno_media_type.workspace = true deno_net.workspace = true deno_package_json.workspace = true +deno_path_util.workspace = true deno_permissions.workspace = true deno_whoami = "0.1.0" der = { version = "0.7.9", features = ["derive"] } @@ -52,7 +54,7 @@ http.workspace = true http-body-util.workspace = true hyper.workspace = true hyper-util.workspace = true -idna = "0.3.0" +idna = "1.0.3" indexmap.workspace = true ipnetwork = "0.20.0" k256 = "0.13.1" @@ -87,12 +89,13 @@ sha1.workspace = true sha2.workspace = true sha3 = { version = "0.10.8", features = ["oid"] } signature.workspace = true -simd-json = "0.13.4" +simd-json = "0.14.0" sm3 = "0.4.2" spki.workspace = true stable_deref_trait = "1.2.0" thiserror.workspace = true tokio.workspace = true +tokio-eld = "0.2" url.workspace = true webpki-root-certs.workspace = true winapi.workspace = true diff --git a/ext/node/lib.rs b/ext/node/lib.rs index af14e3e854b15f..9986b0f607f55e 100644 --- a/ext/node/lib.rs +++ b/ext/node/lib.rs @@ -9,26 +9,23 @@ use std::path::Path; use std::path::PathBuf; use deno_core::error::AnyError; -use deno_core::located_script_name; use deno_core::op2; use deno_core::url::Url; #[allow(unused_imports)] use deno_core::v8; use deno_core::v8::ExternalReference; -use deno_core::JsRuntime; -use deno_core::OpState; -use deno_fs::sync::MaybeSend; -use deno_fs::sync::MaybeSync; -use node_resolver::NpmResolverRc; +use node_resolver::errors::ClosestPkgJsonError; +use node_resolver::NpmPackageFolderResolverRc; use once_cell::sync::Lazy; extern crate libz_sys as zlib; mod global; -mod ops; +pub mod ops; mod polyfill; pub use deno_package_json::PackageJson; +use deno_permissions::PermissionCheckError; pub use node_resolver::PathClean; pub use ops::ipc::ChildPipeFd; pub use ops::ipc::IpcJsonStreamResource; @@ -50,10 +47,18 @@ pub trait NodePermissions { &mut self, url: &Url, api_name: &str, - ) -> Result<(), AnyError>; + ) -> Result<(), PermissionCheckError>; + fn check_net( + &mut self, + host: (&str, Option), + api_name: &str, + ) -> Result<(), PermissionCheckError>; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] #[inline(always)] - fn check_read(&mut self, path: &str) -> Result { + fn check_read( + &mut self, + path: &str, + ) -> Result { self.check_read_with_api_name(path, None) } #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] @@ -61,19 +66,24 @@ pub trait NodePermissions { &mut self, path: &str, api_name: Option<&str>, - ) -> Result; + ) -> Result; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] fn check_read_path<'a>( &mut self, path: &'a Path, - ) -> Result, AnyError>; - fn check_sys(&mut self, kind: &str, api_name: &str) -> Result<(), AnyError>; + ) -> Result, PermissionCheckError>; + fn query_read_all(&mut self) -> bool; + fn check_sys( + &mut self, + kind: &str, + api_name: &str, + ) -> Result<(), PermissionCheckError>; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] fn check_write_with_api_name( &mut self, path: &str, api_name: Option<&str>, - ) -> Result; + ) -> Result; } impl NodePermissions for deno_permissions::PermissionsContainer { @@ -82,16 +92,24 @@ impl NodePermissions for deno_permissions::PermissionsContainer { &mut self, url: &Url, api_name: &str, - ) -> Result<(), AnyError> { + ) -> Result<(), PermissionCheckError> { deno_permissions::PermissionsContainer::check_net_url(self, url, api_name) } + fn check_net( + &mut self, + host: (&str, Option), + api_name: &str, + ) -> Result<(), PermissionCheckError> { + deno_permissions::PermissionsContainer::check_net(self, &host, api_name) + } + #[inline(always)] fn check_read_with_api_name( &mut self, path: &str, api_name: Option<&str>, - ) -> Result { + ) -> Result { deno_permissions::PermissionsContainer::check_read_with_api_name( self, path, api_name, ) @@ -100,54 +118,53 @@ impl NodePermissions for deno_permissions::PermissionsContainer { fn check_read_path<'a>( &mut self, path: &'a Path, - ) -> Result, AnyError> { + ) -> Result, PermissionCheckError> { deno_permissions::PermissionsContainer::check_read_path(self, path, None) } + fn query_read_all(&mut self) -> bool { + deno_permissions::PermissionsContainer::query_read_all(self) + } + #[inline(always)] fn check_write_with_api_name( &mut self, path: &str, api_name: Option<&str>, - ) -> Result { + ) -> Result { deno_permissions::PermissionsContainer::check_write_with_api_name( self, path, api_name, ) } - fn check_sys(&mut self, kind: &str, api_name: &str) -> Result<(), AnyError> { + fn check_sys( + &mut self, + kind: &str, + api_name: &str, + ) -> Result<(), PermissionCheckError> { deno_permissions::PermissionsContainer::check_sys(self, kind, api_name) } } #[allow(clippy::disallowed_types)] -pub type NpmProcessStateProviderRc = - deno_fs::sync::MaybeArc; - -pub trait NpmProcessStateProvider: - std::fmt::Debug + MaybeSend + MaybeSync -{ - /// Gets a string containing the serialized npm state of the process. - /// - /// This will be set on the `DENO_DONT_USE_INTERNAL_NODE_COMPAT_STATE` environment - /// variable when doing a `child_process.fork`. The implementor can then check this environment - /// variable on startup to repopulate the internal npm state. - fn get_npm_process_state(&self) -> String { - // This method is only used in the CLI. - String::new() - } -} +pub type NodeRequireLoaderRc = std::rc::Rc; -#[allow(clippy::disallowed_types)] -pub type NodeRequireResolverRc = - deno_fs::sync::MaybeArc; - -pub trait NodeRequireResolver: std::fmt::Debug + MaybeSend + MaybeSync { - fn ensure_read_permission( +pub trait NodeRequireLoader { + #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] + fn ensure_read_permission<'a>( &self, permissions: &mut dyn NodePermissions, + path: &'a Path, + ) -> Result, AnyError>; + + fn load_text_file_lossy( + &self, path: &Path, - ) -> Result<(), AnyError>; + ) -> Result, AnyError>; + + /// Get if the module kind is maybe CJS and loading should determine + /// if its CJS or ESM. + fn is_maybe_cjs(&self, specifier: &Url) -> Result; } pub static NODE_ENV_VAR_ALLOWLIST: Lazy> = Lazy::new(|| { @@ -165,18 +182,12 @@ fn op_node_build_os() -> String { env!("TARGET").split('-').nth(2).unwrap().to_string() } -#[op2] -#[string] -fn op_npm_process_state(state: &mut OpState) -> Result { - let npm_resolver = state.borrow_mut::(); - Ok(npm_resolver.get_npm_process_state()) -} - +#[derive(Clone)] pub struct NodeExtInitServices { - pub node_require_resolver: NodeRequireResolverRc, + pub node_require_loader: NodeRequireLoaderRc, pub node_resolver: NodeResolverRc, - pub npm_process_state_provider: NpmProcessStateProviderRc, - pub npm_resolver: NpmResolverRc, + pub npm_resolver: NpmPackageFolderResolverRc, + pub pkg_json_resolver: PackageJsonResolverRc, } deno_core::extension!(deno_node, @@ -194,6 +205,7 @@ deno_core::extension!(deno_node, ops::buffer::op_is_ascii, ops::buffer::op_is_utf8, + ops::buffer::op_transcode, ops::crypto::op_node_check_prime_async, ops::crypto::op_node_check_prime_bytes_async, ops::crypto::op_node_check_prime_bytes, @@ -341,6 +353,7 @@ deno_core::extension!(deno_node, ops::zlib::op_zlib_write, ops::zlib::op_zlib_init, ops::zlib::op_zlib_reset, + ops::zlib::op_zlib_crc32, ops::zlib::brotli::op_brotli_compress, ops::zlib::brotli::op_brotli_compress_async, ops::zlib::brotli::op_create_brotli_compress, @@ -368,19 +381,19 @@ deno_core::extension!(deno_node, ops::http2::op_http2_send_response, ops::os::op_node_os_get_priority

, ops::os::op_node_os_set_priority

, - ops::os::op_node_os_username

, + ops::os::op_node_os_user_info

, ops::os::op_geteuid

, ops::os::op_getegid

, ops::os::op_cpus

, ops::os::op_homedir

, op_node_build_os, - op_npm_process_state, ops::require::op_require_can_parse_as_esm, ops::require::op_require_init_paths, ops::require::op_require_node_module_paths

, ops::require::op_require_proxy_path, ops::require::op_require_is_deno_dir_package, ops::require::op_require_resolve_deno_dir, + ops::require::op_require_is_maybe_cjs, ops::require::op_require_is_request_relative, ops::require::op_require_resolve_lookup_paths, ops::require::op_require_try_self_parent_path

, @@ -394,7 +407,6 @@ deno_core::extension!(deno_node, ops::require::op_require_read_file

, ops::require::op_require_as_file_path, ops::require::op_require_resolve_exports

, - ops::require::op_require_read_closest_package_json

, ops::require::op_require_read_package_scope

, ops::require::op_require_package_imports_resolve

, ops::require::op_require_break_on_next_statement, @@ -408,6 +420,18 @@ deno_core::extension!(deno_node, ops::process::op_node_process_kill, ops::process::op_process_abort, ops::tls::op_get_root_certificates, + ops::inspector::op_inspector_open

, + ops::inspector::op_inspector_close, + ops::inspector::op_inspector_url, + ops::inspector::op_inspector_wait, + ops::inspector::op_inspector_connect

, + ops::inspector::op_inspector_dispatch, + ops::inspector::op_inspector_disconnect, + ops::inspector::op_inspector_emit_protocol_event, + ops::inspector::op_inspector_enabled, + ], + objects = [ + ops::perf_hooks::EldHistogram ], esm_entry_point = "ext:deno_node/02_init.js", esm = [ @@ -490,6 +514,7 @@ deno_core::extension!(deno_node, "internal_binding/constants.ts", "internal_binding/crypto.ts", "internal_binding/handle_wrap.ts", + "internal_binding/http_parser.ts", "internal_binding/mod.ts", "internal_binding/node_file.ts", "internal_binding/node_options.ts", @@ -615,8 +640,8 @@ deno_core::extension!(deno_node, "node:http" = "http.ts", "node:http2" = "http2.ts", "node:https" = "https.ts", - "node:inspector" = "inspector.ts", - "node:inspector/promises" = "inspector.ts", + "node:inspector" = "inspector.js", + "node:inspector/promises" = "inspector/promises.js", "node:module" = "01_require.js", "node:net" = "net.ts", "node:os" = "os.ts", @@ -659,10 +684,10 @@ deno_core::extension!(deno_node, state.put(options.fs.clone()); if let Some(init) = &options.maybe_init { - state.put(init.node_require_resolver.clone()); + state.put(init.node_require_loader.clone()); state.put(init.node_resolver.clone()); state.put(init.npm_resolver.clone()); - state.put(init.npm_process_state_provider.clone()); + state.put(init.pkg_json_resolver.clone()); } }, global_template_middleware = global_template_middleware, @@ -782,33 +807,16 @@ deno_core::extension!(deno_node, }, ); -pub fn load_cjs_module( - js_runtime: &mut JsRuntime, - module: &str, - main: bool, - inspect_brk: bool, -) -> Result<(), AnyError> { - fn escape_for_single_quote_string(text: &str) -> String { - text.replace('\\', r"\\").replace('\'', r"\'") - } - - let source_code = format!( - r#"(function loadCjsModule(moduleName, isMain, inspectBrk) {{ - Deno[Deno.internal].node.loadCjsModule(moduleName, isMain, inspectBrk); - }})('{module}', {main}, {inspect_brk});"#, - main = main, - module = escape_for_single_quote_string(module), - inspect_brk = inspect_brk, - ); - - js_runtime.execute_script(located_script_name!(), source_code)?; - Ok(()) -} - pub type NodeResolver = node_resolver::NodeResolver; #[allow(clippy::disallowed_types)] pub type NodeResolverRc = deno_fs::sync::MaybeArc>; +pub type PackageJsonResolver = + node_resolver::PackageJsonResolver; +#[allow(clippy::disallowed_types)] +pub type PackageJsonResolverRc = deno_fs::sync::MaybeArc< + node_resolver::PackageJsonResolver, +>; #[derive(Debug)] pub struct DenoFsNodeResolverEnv { @@ -868,6 +876,8 @@ impl deno_package_json::fs::DenoPkgJsonFs for DenoFsNodeResolverEnv { self .fs .read_text_file_lossy_sync(path, None) + // todo(https://github.com/denoland/deno_package_json/pull/9): don't clone + .map(|text| text.into_owned()) .map_err(|err| err.into_io_error()) } } @@ -882,6 +892,8 @@ impl<'a> deno_package_json::fs::DenoPkgJsonFs for DenoPkgJsonFsAdapter<'a> { self .0 .read_text_file_lossy_sync(path, None) + // todo(https://github.com/denoland/deno_package_json/pull/9): don't clone + .map(|text| text.into_owned()) .map_err(|err| err.into_io_error()) } } diff --git a/ext/node/ops/blocklist.rs b/ext/node/ops/blocklist.rs index 332cdda8f93549..6c64d68ecae653 100644 --- a/ext/node/ops/blocklist.rs +++ b/ext/node/ops/blocklist.rs @@ -7,9 +7,6 @@ use std::net::Ipv4Addr; use std::net::Ipv6Addr; use std::net::SocketAddr; -use deno_core::anyhow::anyhow; -use deno_core::anyhow::bail; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; @@ -27,13 +24,25 @@ impl deno_core::GarbageCollected for BlockListResource {} #[derive(Serialize)] struct SocketAddressSerialization(String, String); +#[derive(Debug, thiserror::Error)] +pub enum BlocklistError { + #[error("{0}")] + AddrParse(#[from] std::net::AddrParseError), + #[error("{0}")] + IpNetwork(#[from] ipnetwork::IpNetworkError), + #[error("Invalid address")] + InvalidAddress, + #[error("IP version mismatch between start and end addresses")] + IpVersionMismatch, +} + #[op2(fast)] pub fn op_socket_address_parse( state: &mut OpState, #[string] addr: &str, #[smi] port: u16, #[string] family: &str, -) -> Result { +) -> Result { let ip = addr.parse::()?; let parsed: SocketAddr = SocketAddr::new(ip, port); let parsed_ip_str = parsed.ip().to_string(); @@ -52,7 +61,7 @@ pub fn op_socket_address_parse( Ok(false) } } else { - Err(anyhow!("Invalid address")) + Err(BlocklistError::InvalidAddress) } } @@ -60,8 +69,8 @@ pub fn op_socket_address_parse( #[serde] pub fn op_socket_address_get_serialization( state: &mut OpState, -) -> Result { - Ok(state.take::()) +) -> SocketAddressSerialization { + state.take::() } #[op2] @@ -77,7 +86,7 @@ pub fn op_blocklist_new() -> BlockListResource { pub fn op_blocklist_add_address( #[cppgc] wrap: &BlockListResource, #[string] addr: &str, -) -> Result<(), AnyError> { +) -> Result<(), BlocklistError> { wrap.blocklist.borrow_mut().add_address(addr) } @@ -86,7 +95,7 @@ pub fn op_blocklist_add_range( #[cppgc] wrap: &BlockListResource, #[string] start: &str, #[string] end: &str, -) -> Result { +) -> Result { wrap.blocklist.borrow_mut().add_range(start, end) } @@ -95,7 +104,7 @@ pub fn op_blocklist_add_subnet( #[cppgc] wrap: &BlockListResource, #[string] addr: &str, #[smi] prefix: u8, -) -> Result<(), AnyError> { +) -> Result<(), BlocklistError> { wrap.blocklist.borrow_mut().add_subnet(addr, prefix) } @@ -104,7 +113,7 @@ pub fn op_blocklist_check( #[cppgc] wrap: &BlockListResource, #[string] addr: &str, #[string] r#type: &str, -) -> Result { +) -> Result { wrap.blocklist.borrow().check(addr, r#type) } @@ -123,7 +132,7 @@ impl BlockList { &mut self, addr: IpAddr, prefix: Option, - ) -> Result<(), AnyError> { + ) -> Result<(), BlocklistError> { match addr { IpAddr::V4(addr) => { let ipv4_prefix = prefix.unwrap_or(32); @@ -154,7 +163,7 @@ impl BlockList { Ok(()) } - pub fn add_address(&mut self, address: &str) -> Result<(), AnyError> { + pub fn add_address(&mut self, address: &str) -> Result<(), BlocklistError> { let ip: IpAddr = address.parse()?; self.map_addr_add_network(ip, None)?; Ok(()) @@ -164,7 +173,7 @@ impl BlockList { &mut self, start: &str, end: &str, - ) -> Result { + ) -> Result { let start_ip: IpAddr = start.parse()?; let end_ip: IpAddr = end.parse()?; @@ -193,25 +202,33 @@ impl BlockList { self.map_addr_add_network(IpAddr::V6(addr), None)?; } } - _ => bail!("IP version mismatch between start and end addresses"), + _ => return Err(BlocklistError::IpVersionMismatch), } Ok(true) } - pub fn add_subnet(&mut self, addr: &str, prefix: u8) -> Result<(), AnyError> { + pub fn add_subnet( + &mut self, + addr: &str, + prefix: u8, + ) -> Result<(), BlocklistError> { let ip: IpAddr = addr.parse()?; self.map_addr_add_network(ip, Some(prefix))?; Ok(()) } - pub fn check(&self, addr: &str, r#type: &str) -> Result { + pub fn check( + &self, + addr: &str, + r#type: &str, + ) -> Result { let addr: IpAddr = addr.parse()?; let family = r#type.to_lowercase(); if family == "ipv4" && addr.is_ipv4() || family == "ipv6" && addr.is_ipv6() { Ok(self.rules.iter().any(|net| net.contains(addr))) } else { - Err(anyhow!("Invalid address")) + Err(BlocklistError::InvalidAddress) } } } diff --git a/ext/node/ops/buffer.rs b/ext/node/ops/buffer.rs index 74a011ab80989d..01f878ec15bdd2 100644 --- a/ext/node/ops/buffer.rs +++ b/ext/node/ops/buffer.rs @@ -1,5 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use deno_core::anyhow::anyhow; +use deno_core::anyhow::Result; use deno_core::op2; #[op2(fast)] @@ -11,3 +13,107 @@ pub fn op_is_ascii(#[buffer] buf: &[u8]) -> bool { pub fn op_is_utf8(#[buffer] buf: &[u8]) -> bool { std::str::from_utf8(buf).is_ok() } + +#[op2] +#[buffer] +pub fn op_transcode( + #[buffer] source: &[u8], + #[string] from_encoding: &str, + #[string] to_encoding: &str, +) -> Result> { + match (from_encoding, to_encoding) { + ("utf8", "ascii") => Ok(utf8_to_ascii(source)), + ("utf8", "latin1") => Ok(utf8_to_latin1(source)), + ("utf8", "utf16le") => utf8_to_utf16le(source), + ("utf16le", "utf8") => utf16le_to_utf8(source), + ("latin1", "utf16le") | ("ascii", "utf16le") => { + Ok(latin1_ascii_to_utf16le(source)) + } + (from, to) => Err(anyhow!("Unable to transcode Buffer {from}->{to}")), + } +} + +fn latin1_ascii_to_utf16le(source: &[u8]) -> Vec { + let mut result = Vec::with_capacity(source.len() * 2); + for &byte in source { + result.push(byte); + result.push(0); + } + result +} + +fn utf16le_to_utf8(source: &[u8]) -> Result> { + let ucs2_vec: Vec = source + .chunks(2) + .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]])) + .collect(); + String::from_utf16(&ucs2_vec) + .map(|utf8_string| utf8_string.into_bytes()) + .map_err(|e| anyhow!("Invalid UTF-16 sequence: {}", e)) +} + +fn utf8_to_utf16le(source: &[u8]) -> Result> { + let utf8_string = std::str::from_utf8(source)?; + let ucs2_vec: Vec = utf8_string.encode_utf16().collect(); + let bytes: Vec = ucs2_vec.iter().flat_map(|&x| x.to_le_bytes()).collect(); + Ok(bytes) +} + +fn utf8_to_latin1(source: &[u8]) -> Vec { + let mut latin1_bytes = Vec::with_capacity(source.len()); + let mut i = 0; + while i < source.len() { + match source[i] { + byte if byte <= 0x7F => { + // ASCII character + latin1_bytes.push(byte); + i += 1; + } + byte if (0xC2..=0xDF).contains(&byte) && i + 1 < source.len() => { + // 2-byte UTF-8 sequence + let codepoint = + ((byte as u16 & 0x1F) << 6) | (source[i + 1] as u16 & 0x3F); + latin1_bytes.push(if codepoint <= 0xFF { + codepoint as u8 + } else { + b'?' + }); + i += 2; + } + _ => { + // 3-byte or 4-byte UTF-8 sequence, or invalid UTF-8 + latin1_bytes.push(b'?'); + // Skip to the next valid UTF-8 start byte + i += 1; + while i < source.len() && (source[i] & 0xC0) == 0x80 { + i += 1; + } + } + } + } + latin1_bytes +} + +fn utf8_to_ascii(source: &[u8]) -> Vec { + let mut ascii_bytes = Vec::with_capacity(source.len()); + let mut i = 0; + while i < source.len() { + match source[i] { + byte if byte <= 0x7F => { + // ASCII character + ascii_bytes.push(byte); + i += 1; + } + _ => { + // Non-ASCII character + ascii_bytes.push(b'?'); + // Skip to the next valid UTF-8 start byte + i += 1; + while i < source.len() && (source[i] & 0xC0) == 0x80 { + i += 1; + } + } + } + } + ascii_bytes +} diff --git a/ext/node/ops/crypto/cipher.rs b/ext/node/ops/crypto/cipher.rs index b80aa33fe8352f..ec45146b49aeaa 100644 --- a/ext/node/ops/crypto/cipher.rs +++ b/ext/node/ops/crypto/cipher.rs @@ -4,9 +4,6 @@ use aes::cipher::block_padding::Pkcs7; use aes::cipher::BlockDecryptMut; use aes::cipher::BlockEncryptMut; use aes::cipher::KeyIvInit; -use deno_core::error::range_error; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::Resource; use digest::generic_array::GenericArray; use digest::KeyInit; @@ -50,8 +47,22 @@ pub struct DecipherContext { decipher: Rc>, } +#[derive(Debug, thiserror::Error)] +pub enum CipherContextError { + #[error("Cipher context is already in use")] + ContextInUse, + #[error("{0}")] + Resource(deno_core::error::AnyError), + #[error(transparent)] + Cipher(#[from] CipherError), +} + impl CipherContext { - pub fn new(algorithm: &str, key: &[u8], iv: &[u8]) -> Result { + pub fn new( + algorithm: &str, + key: &[u8], + iv: &[u8], + ) -> Result { Ok(Self { cipher: Rc::new(RefCell::new(Cipher::new(algorithm, key, iv)?)), }) @@ -74,16 +85,31 @@ impl CipherContext { auto_pad: bool, input: &[u8], output: &mut [u8], - ) -> Result { + ) -> Result { Rc::try_unwrap(self.cipher) - .map_err(|_| type_error("Cipher context is already in use"))? + .map_err(|_| CipherContextError::ContextInUse)? .into_inner() .r#final(auto_pad, input, output) + .map_err(Into::into) } } +#[derive(Debug, thiserror::Error)] +pub enum DecipherContextError { + #[error("Decipher context is already in use")] + ContextInUse, + #[error("{0}")] + Resource(deno_core::error::AnyError), + #[error(transparent)] + Decipher(#[from] DecipherError), +} + impl DecipherContext { - pub fn new(algorithm: &str, key: &[u8], iv: &[u8]) -> Result { + pub fn new( + algorithm: &str, + key: &[u8], + iv: &[u8], + ) -> Result { Ok(Self { decipher: Rc::new(RefCell::new(Decipher::new(algorithm, key, iv)?)), }) @@ -103,11 +129,12 @@ impl DecipherContext { input: &[u8], output: &mut [u8], auth_tag: &[u8], - ) -> Result<(), AnyError> { + ) -> Result<(), DecipherContextError> { Rc::try_unwrap(self.decipher) - .map_err(|_| type_error("Decipher context is already in use"))? + .map_err(|_| DecipherContextError::ContextInUse)? .into_inner() .r#final(auto_pad, input, output, auth_tag) + .map_err(Into::into) } } @@ -123,12 +150,26 @@ impl Resource for DecipherContext { } } +#[derive(Debug, thiserror::Error)] +pub enum CipherError { + #[error("IV length must be 12 bytes")] + InvalidIvLength, + #[error("Invalid key length")] + InvalidKeyLength, + #[error("Invalid initialization vector")] + InvalidInitializationVector, + #[error("Cannot pad the input data")] + CannotPadInputData, + #[error("Unknown cipher {0}")] + UnknownCipher(String), +} + impl Cipher { fn new( algorithm_name: &str, key: &[u8], iv: &[u8], - ) -> Result { + ) -> Result { use Cipher::*; Ok(match algorithm_name { "aes-128-cbc" => { @@ -139,7 +180,7 @@ impl Cipher { "aes-256-ecb" => Aes256Ecb(Box::new(ecb::Encryptor::new(key.into()))), "aes-128-gcm" => { if iv.len() != 12 { - return Err(type_error("IV length must be 12 bytes")); + return Err(CipherError::InvalidIvLength); } let cipher = @@ -149,7 +190,7 @@ impl Cipher { } "aes-256-gcm" => { if iv.len() != 12 { - return Err(type_error("IV length must be 12 bytes")); + return Err(CipherError::InvalidIvLength); } let cipher = @@ -159,15 +200,15 @@ impl Cipher { } "aes256" | "aes-256-cbc" => { if key.len() != 32 { - return Err(range_error("Invalid key length")); + return Err(CipherError::InvalidKeyLength); } if iv.len() != 16 { - return Err(type_error("Invalid initialization vector")); + return Err(CipherError::InvalidInitializationVector); } Aes256Cbc(Box::new(cbc::Encryptor::new(key.into(), iv.into()))) } - _ => return Err(type_error(format!("Unknown cipher {algorithm_name}"))), + _ => return Err(CipherError::UnknownCipher(algorithm_name.to_string())), }) } @@ -235,14 +276,14 @@ impl Cipher { auto_pad: bool, input: &[u8], output: &mut [u8], - ) -> Result { + ) -> Result { assert!(input.len() < 16); use Cipher::*; match (self, auto_pad) { (Aes128Cbc(encryptor), true) => { let _ = (*encryptor) .encrypt_padded_b2b_mut::(input, output) - .map_err(|_| type_error("Cannot pad the input data"))?; + .map_err(|_| CipherError::CannotPadInputData)?; Ok(None) } (Aes128Cbc(mut encryptor), false) => { @@ -255,7 +296,7 @@ impl Cipher { (Aes128Ecb(encryptor), true) => { let _ = (*encryptor) .encrypt_padded_b2b_mut::(input, output) - .map_err(|_| type_error("Cannot pad the input data"))?; + .map_err(|_| CipherError::CannotPadInputData)?; Ok(None) } (Aes128Ecb(mut encryptor), false) => { @@ -268,7 +309,7 @@ impl Cipher { (Aes192Ecb(encryptor), true) => { let _ = (*encryptor) .encrypt_padded_b2b_mut::(input, output) - .map_err(|_| type_error("Cannot pad the input data"))?; + .map_err(|_| CipherError::CannotPadInputData)?; Ok(None) } (Aes192Ecb(mut encryptor), false) => { @@ -281,7 +322,7 @@ impl Cipher { (Aes256Ecb(encryptor), true) => { let _ = (*encryptor) .encrypt_padded_b2b_mut::(input, output) - .map_err(|_| type_error("Cannot pad the input data"))?; + .map_err(|_| CipherError::CannotPadInputData)?; Ok(None) } (Aes256Ecb(mut encryptor), false) => { @@ -296,7 +337,7 @@ impl Cipher { (Aes256Cbc(encryptor), true) => { let _ = (*encryptor) .encrypt_padded_b2b_mut::(input, output) - .map_err(|_| type_error("Cannot pad the input data"))?; + .map_err(|_| CipherError::CannotPadInputData)?; Ok(None) } (Aes256Cbc(mut encryptor), false) => { @@ -319,12 +360,32 @@ impl Cipher { } } +#[derive(Debug, thiserror::Error)] +pub enum DecipherError { + #[error("IV length must be 12 bytes")] + InvalidIvLength, + #[error("Invalid key length")] + InvalidKeyLength, + #[error("Invalid initialization vector")] + InvalidInitializationVector, + #[error("Cannot unpad the input data")] + CannotUnpadInputData, + #[error("Failed to authenticate data")] + DataAuthenticationFailed, + #[error("setAutoPadding(false) not supported for Aes128Gcm yet")] + SetAutoPaddingFalseAes128GcmUnsupported, + #[error("setAutoPadding(false) not supported for Aes256Gcm yet")] + SetAutoPaddingFalseAes256GcmUnsupported, + #[error("Unknown cipher {0}")] + UnknownCipher(String), +} + impl Decipher { fn new( algorithm_name: &str, key: &[u8], iv: &[u8], - ) -> Result { + ) -> Result { use Decipher::*; Ok(match algorithm_name { "aes-128-cbc" => { @@ -335,7 +396,7 @@ impl Decipher { "aes-256-ecb" => Aes256Ecb(Box::new(ecb::Decryptor::new(key.into()))), "aes-128-gcm" => { if iv.len() != 12 { - return Err(type_error("IV length must be 12 bytes")); + return Err(DecipherError::InvalidIvLength); } let decipher = @@ -345,7 +406,7 @@ impl Decipher { } "aes-256-gcm" => { if iv.len() != 12 { - return Err(type_error("IV length must be 12 bytes")); + return Err(DecipherError::InvalidIvLength); } let decipher = @@ -355,15 +416,17 @@ impl Decipher { } "aes256" | "aes-256-cbc" => { if key.len() != 32 { - return Err(range_error("Invalid key length")); + return Err(DecipherError::InvalidKeyLength); } if iv.len() != 16 { - return Err(type_error("Invalid initialization vector")); + return Err(DecipherError::InvalidInitializationVector); } Aes256Cbc(Box::new(cbc::Decryptor::new(key.into(), iv.into()))) } - _ => return Err(type_error(format!("Unknown cipher {algorithm_name}"))), + _ => { + return Err(DecipherError::UnknownCipher(algorithm_name.to_string())) + } }) } @@ -432,14 +495,14 @@ impl Decipher { input: &[u8], output: &mut [u8], auth_tag: &[u8], - ) -> Result<(), AnyError> { + ) -> Result<(), DecipherError> { use Decipher::*; match (self, auto_pad) { (Aes128Cbc(decryptor), true) => { assert!(input.len() == 16); let _ = (*decryptor) .decrypt_padded_b2b_mut::(input, output) - .map_err(|_| type_error("Cannot unpad the input data"))?; + .map_err(|_| DecipherError::CannotUnpadInputData)?; Ok(()) } (Aes128Cbc(mut decryptor), false) => { @@ -453,7 +516,7 @@ impl Decipher { assert!(input.len() == 16); let _ = (*decryptor) .decrypt_padded_b2b_mut::(input, output) - .map_err(|_| type_error("Cannot unpad the input data"))?; + .map_err(|_| DecipherError::CannotUnpadInputData)?; Ok(()) } (Aes128Ecb(mut decryptor), false) => { @@ -467,7 +530,7 @@ impl Decipher { assert!(input.len() == 16); let _ = (*decryptor) .decrypt_padded_b2b_mut::(input, output) - .map_err(|_| type_error("Cannot unpad the input data"))?; + .map_err(|_| DecipherError::CannotUnpadInputData)?; Ok(()) } (Aes192Ecb(mut decryptor), false) => { @@ -481,7 +544,7 @@ impl Decipher { assert!(input.len() == 16); let _ = (*decryptor) .decrypt_padded_b2b_mut::(input, output) - .map_err(|_| type_error("Cannot unpad the input data"))?; + .map_err(|_| DecipherError::CannotUnpadInputData)?; Ok(()) } (Aes256Ecb(mut decryptor), false) => { @@ -496,28 +559,28 @@ impl Decipher { if tag.as_slice() == auth_tag { Ok(()) } else { - Err(type_error("Failed to authenticate data")) + Err(DecipherError::DataAuthenticationFailed) } } - (Aes128Gcm(_), false) => Err(type_error( - "setAutoPadding(false) not supported for Aes256Gcm yet", - )), + (Aes128Gcm(_), false) => { + Err(DecipherError::SetAutoPaddingFalseAes128GcmUnsupported) + } (Aes256Gcm(decipher), true) => { let tag = decipher.finish(); if tag.as_slice() == auth_tag { Ok(()) } else { - Err(type_error("Failed to authenticate data")) + Err(DecipherError::DataAuthenticationFailed) } } - (Aes256Gcm(_), false) => Err(type_error( - "setAutoPadding(false) not supported for Aes256Gcm yet", - )), + (Aes256Gcm(_), false) => { + Err(DecipherError::SetAutoPaddingFalseAes256GcmUnsupported) + } (Aes256Cbc(decryptor), true) => { assert!(input.len() == 16); let _ = (*decryptor) .decrypt_padded_b2b_mut::(input, output) - .map_err(|_| type_error("Cannot unpad the input data"))?; + .map_err(|_| DecipherError::CannotUnpadInputData)?; Ok(()) } (Aes256Cbc(mut decryptor), false) => { diff --git a/ext/node/ops/crypto/digest.rs b/ext/node/ops/crypto/digest.rs index 293e8e063750be..a7d8fb51f1b26c 100644 --- a/ext/node/ops/crypto/digest.rs +++ b/ext/node/ops/crypto/digest.rs @@ -1,6 +1,4 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::generic_error; -use deno_core::error::AnyError; use deno_core::GarbageCollected; use digest::Digest; use digest::DynDigest; @@ -19,7 +17,7 @@ impl Hasher { pub fn new( algorithm: &str, output_length: Option, - ) -> Result { + ) -> Result { let hash = Hash::new(algorithm, output_length)?; Ok(Self { @@ -44,7 +42,7 @@ impl Hasher { pub fn clone_inner( &self, output_length: Option, - ) -> Result, AnyError> { + ) -> Result, HashError> { let hash = self.hash.borrow(); let Some(hash) = hash.as_ref() else { return Ok(None); @@ -184,11 +182,19 @@ pub enum Hash { use Hash::*; +#[derive(Debug, thiserror::Error)] +pub enum HashError { + #[error("Output length mismatch for non-extendable algorithm")] + OutputLengthMismatch, + #[error("Digest method not supported: {0}")] + DigestMethodUnsupported(String), +} + impl Hash { pub fn new( algorithm_name: &str, output_length: Option, - ) -> Result { + ) -> Result { match algorithm_name { "shake128" => return Ok(Shake128(Default::default(), output_length)), "shake256" => return Ok(Shake256(Default::default(), output_length)), @@ -201,17 +207,13 @@ impl Hash { let digest: D = Digest::new(); if let Some(length) = output_length { if length != digest.output_size() { - return Err(generic_error( - "Output length mismatch for non-extendable algorithm", - )); + return Err(HashError::OutputLengthMismatch); } } FixedSize(Box::new(digest)) }, _ => { - return Err(generic_error(format!( - "Digest method not supported: {algorithm_name}" - ))) + return Err(HashError::DigestMethodUnsupported(algorithm_name.to_string())) } ); @@ -243,14 +245,12 @@ impl Hash { pub fn clone_hash( &self, output_length: Option, - ) -> Result { + ) -> Result { let hash = match self { FixedSize(context) => { if let Some(length) = output_length { if length != context.output_size() { - return Err(generic_error( - "Output length mismatch for non-extendable algorithm", - )); + return Err(HashError::OutputLengthMismatch); } } FixedSize(context.box_clone()) diff --git a/ext/node/ops/crypto/keys.rs b/ext/node/ops/crypto/keys.rs index 867b34e04454f7..f164972d48afc6 100644 --- a/ext/node/ops/crypto/keys.rs +++ b/ext/node/ops/crypto/keys.rs @@ -4,9 +4,7 @@ use std::borrow::Cow; use std::cell::RefCell; use base64::Engine; -use deno_core::error::generic_error; use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::serde_v8::BigInt as V8BigInt; use deno_core::unsync::spawn_blocking; @@ -46,6 +44,7 @@ use spki::der::Reader as _; use spki::DecodePublicKey as _; use spki::EncodePublicKey as _; use spki::SubjectPublicKeyInfoRef; +use x509_parser::error::X509Error; use x509_parser::x509; use super::dh; @@ -236,9 +235,11 @@ impl RsaPssPrivateKey { } impl EcPublicKey { - pub fn to_jwk(&self) -> Result { + pub fn to_jwk(&self) -> Result { match self { - EcPublicKey::P224(_) => Err(type_error("Unsupported JWK EC curve: P224")), + EcPublicKey::P224(_) => { + Err(AsymmetricPublicKeyJwkError::UnsupportedJwkEcCurveP224) + } EcPublicKey::P256(key) => Ok(key.to_jwk()), EcPublicKey::P384(key) => Ok(key.to_jwk()), } @@ -363,49 +364,201 @@ impl<'a> TryFrom> for RsaPssParameters<'a> { } } +#[derive(Debug, thiserror::Error)] +pub enum X509PublicKeyError { + #[error(transparent)] + X509(#[from] x509_parser::error::X509Error), + #[error(transparent)] + Rsa(#[from] rsa::Error), + #[error(transparent)] + Asn1(#[from] x509_parser::der_parser::asn1_rs::Error), + #[error(transparent)] + Ec(#[from] elliptic_curve::Error), + #[error("unsupported ec named curve")] + UnsupportedEcNamedCurve, + #[error("missing ec parameters")] + MissingEcParameters, + #[error("malformed DSS public key")] + MalformedDssPublicKey, + #[error("unsupported x509 public key type")] + UnsupportedX509KeyType, +} + +#[derive(Debug, thiserror::Error)] +pub enum RsaJwkError { + #[error(transparent)] + Base64(#[from] base64::DecodeError), + #[error(transparent)] + Rsa(#[from] rsa::Error), + #[error("missing RSA private component")] + MissingRsaPrivateComponent, +} + +#[derive(Debug, thiserror::Error)] +pub enum EcJwkError { + #[error(transparent)] + Ec(#[from] elliptic_curve::Error), + #[error("unsupported curve: {0}")] + UnsupportedCurve(String), +} + +#[derive(Debug, thiserror::Error)] +pub enum EdRawError { + #[error(transparent)] + Ed25519Signature(#[from] ed25519_dalek::SignatureError), + #[error("invalid Ed25519 key")] + InvalidEd25519Key, + #[error("unsupported curve")] + UnsupportedCurve, +} + +#[derive(Debug, thiserror::Error)] +pub enum AsymmetricPrivateKeyError { + #[error("invalid PEM private key: not valid utf8 starting at byte {0}")] + InvalidPemPrivateKeyInvalidUtf8(usize), + #[error("invalid encrypted PEM private key")] + InvalidEncryptedPemPrivateKey, + #[error("invalid PEM private key")] + InvalidPemPrivateKey, + #[error("encrypted private key requires a passphrase to decrypt")] + EncryptedPrivateKeyRequiresPassphraseToDecrypt, + #[error("invalid PKCS#1 private key")] + InvalidPkcs1PrivateKey, + #[error("invalid SEC1 private key")] + InvalidSec1PrivateKey, + #[error("unsupported PEM label: {0}")] + UnsupportedPemLabel(String), + #[error(transparent)] + RsaPssParamsParse(#[from] RsaPssParamsParseError), + #[error("invalid encrypted PKCS#8 private key")] + InvalidEncryptedPkcs8PrivateKey, + #[error("invalid PKCS#8 private key")] + InvalidPkcs8PrivateKey, + #[error("PKCS#1 private key does not support encryption with passphrase")] + Pkcs1PrivateKeyDoesNotSupportEncryptionWithPassphrase, + #[error("SEC1 private key does not support encryption with passphrase")] + Sec1PrivateKeyDoesNotSupportEncryptionWithPassphrase, + #[error("unsupported ec named curve")] + UnsupportedEcNamedCurve, + #[error("invalid private key")] + InvalidPrivateKey, + #[error("invalid DSA private key")] + InvalidDsaPrivateKey, + #[error("malformed or missing named curve in ec parameters")] + MalformedOrMissingNamedCurveInEcParameters, + #[error("unsupported key type: {0}")] + UnsupportedKeyType(String), + #[error("unsupported key format: {0}")] + UnsupportedKeyFormat(String), + #[error("invalid x25519 private key")] + InvalidX25519PrivateKey, + #[error("x25519 private key is the wrong length")] + X25519PrivateKeyIsWrongLength, + #[error("invalid Ed25519 private key")] + InvalidEd25519PrivateKey, + #[error("missing dh parameters")] + MissingDhParameters, + #[error("unsupported private key oid")] + UnsupportedPrivateKeyOid, +} + +#[derive(Debug, thiserror::Error)] +pub enum AsymmetricPublicKeyError { + #[error("invalid PEM private key: not valid utf8 starting at byte {0}")] + InvalidPemPrivateKeyInvalidUtf8(usize), + #[error("invalid PEM public key")] + InvalidPemPublicKey, + #[error("invalid PKCS#1 public key")] + InvalidPkcs1PublicKey, + #[error(transparent)] + AsymmetricPrivateKey(#[from] AsymmetricPrivateKeyError), + #[error("invalid x509 certificate")] + InvalidX509Certificate, + #[error(transparent)] + X509(#[from] x509_parser::nom::Err), + #[error(transparent)] + X509PublicKey(#[from] X509PublicKeyError), + #[error("unsupported PEM label: {0}")] + UnsupportedPemLabel(String), + #[error("invalid SPKI public key")] + InvalidSpkiPublicKey, + #[error("unsupported key type: {0}")] + UnsupportedKeyType(String), + #[error("unsupported key format: {0}")] + UnsupportedKeyFormat(String), + #[error(transparent)] + Spki(#[from] spki::Error), + #[error(transparent)] + Pkcs1(#[from] rsa::pkcs1::Error), + #[error(transparent)] + RsaPssParamsParse(#[from] RsaPssParamsParseError), + #[error("malformed DSS public key")] + MalformedDssPublicKey, + #[error("malformed or missing named curve in ec parameters")] + MalformedOrMissingNamedCurveInEcParameters, + #[error("malformed or missing public key in ec spki")] + MalformedOrMissingPublicKeyInEcSpki, + #[error(transparent)] + Ec(#[from] elliptic_curve::Error), + #[error("unsupported ec named curve")] + UnsupportedEcNamedCurve, + #[error("malformed or missing public key in x25519 spki")] + MalformedOrMissingPublicKeyInX25519Spki, + #[error("x25519 public key is too short")] + X25519PublicKeyIsTooShort, + #[error("invalid Ed25519 public key")] + InvalidEd25519PublicKey, + #[error("missing dh parameters")] + MissingDhParameters, + #[error("malformed dh parameters")] + MalformedDhParameters, + #[error("malformed or missing public key in dh spki")] + MalformedOrMissingPublicKeyInDhSpki, + #[error("unsupported private key oid")] + UnsupportedPrivateKeyOid, +} + impl KeyObjectHandle { pub fn new_asymmetric_private_key_from_js( key: &[u8], format: &str, typ: &str, passphrase: Option<&[u8]>, - ) -> Result { + ) -> Result { let document = match format { "pem" => { let pem = std::str::from_utf8(key).map_err(|err| { - type_error(format!( - "invalid PEM private key: not valid utf8 starting at byte {}", - err.valid_up_to() - )) + AsymmetricPrivateKeyError::InvalidPemPrivateKeyInvalidUtf8( + err.valid_up_to(), + ) })?; if let Some(passphrase) = passphrase { - SecretDocument::from_pkcs8_encrypted_pem(pem, passphrase) - .map_err(|_| type_error("invalid encrypted PEM private key"))? + SecretDocument::from_pkcs8_encrypted_pem(pem, passphrase).map_err( + |_| AsymmetricPrivateKeyError::InvalidEncryptedPemPrivateKey, + )? } else { let (label, doc) = SecretDocument::from_pem(pem) - .map_err(|_| type_error("invalid PEM private key"))?; + .map_err(|_| AsymmetricPrivateKeyError::InvalidPemPrivateKey)?; match label { EncryptedPrivateKeyInfo::PEM_LABEL => { - return Err(type_error( - "encrypted private key requires a passphrase to decrypt", - )) + return Err(AsymmetricPrivateKeyError::EncryptedPrivateKeyRequiresPassphraseToDecrypt); } PrivateKeyInfo::PEM_LABEL => doc, rsa::pkcs1::RsaPrivateKey::PEM_LABEL => { - SecretDocument::from_pkcs1_der(doc.as_bytes()) - .map_err(|_| type_error("invalid PKCS#1 private key"))? + SecretDocument::from_pkcs1_der(doc.as_bytes()).map_err(|_| { + AsymmetricPrivateKeyError::InvalidPkcs1PrivateKey + })? } sec1::EcPrivateKey::PEM_LABEL => { SecretDocument::from_sec1_der(doc.as_bytes()) - .map_err(|_| type_error("invalid SEC1 private key"))? + .map_err(|_| AsymmetricPrivateKeyError::InvalidSec1PrivateKey)? } _ => { - return Err(type_error(format!( - "unsupported PEM label: {}", - label - ))) + return Err(AsymmetricPrivateKeyError::UnsupportedPemLabel( + label.to_string(), + )) } } } @@ -413,54 +566,57 @@ impl KeyObjectHandle { "der" => match typ { "pkcs8" => { if let Some(passphrase) = passphrase { - SecretDocument::from_pkcs8_encrypted_der(key, passphrase) - .map_err(|_| type_error("invalid encrypted PKCS#8 private key"))? + SecretDocument::from_pkcs8_encrypted_der(key, passphrase).map_err( + |_| AsymmetricPrivateKeyError::InvalidEncryptedPkcs8PrivateKey, + )? } else { SecretDocument::from_pkcs8_der(key) - .map_err(|_| type_error("invalid PKCS#8 private key"))? + .map_err(|_| AsymmetricPrivateKeyError::InvalidPkcs8PrivateKey)? } } "pkcs1" => { if passphrase.is_some() { - return Err(type_error( - "PKCS#1 private key does not support encryption with passphrase", - )); + return Err(AsymmetricPrivateKeyError::Pkcs1PrivateKeyDoesNotSupportEncryptionWithPassphrase); } SecretDocument::from_pkcs1_der(key) - .map_err(|_| type_error("invalid PKCS#1 private key"))? + .map_err(|_| AsymmetricPrivateKeyError::InvalidPkcs1PrivateKey)? } "sec1" => { if passphrase.is_some() { - return Err(type_error( - "SEC1 private key does not support encryption with passphrase", - )); + return Err(AsymmetricPrivateKeyError::Sec1PrivateKeyDoesNotSupportEncryptionWithPassphrase); } SecretDocument::from_sec1_der(key) - .map_err(|_| type_error("invalid SEC1 private key"))? + .map_err(|_| AsymmetricPrivateKeyError::InvalidSec1PrivateKey)? + } + _ => { + return Err(AsymmetricPrivateKeyError::UnsupportedKeyType( + typ.to_string(), + )) } - _ => return Err(type_error(format!("unsupported key type: {}", typ))), }, _ => { - return Err(type_error(format!("unsupported key format: {}", format))) + return Err(AsymmetricPrivateKeyError::UnsupportedKeyFormat( + format.to_string(), + )) } }; let pk_info = PrivateKeyInfo::try_from(document.as_bytes()) - .map_err(|_| type_error("invalid private key"))?; + .map_err(|_| AsymmetricPrivateKeyError::InvalidPrivateKey)?; let alg = pk_info.algorithm.oid; let private_key = match alg { RSA_ENCRYPTION_OID => { let private_key = rsa::RsaPrivateKey::from_pkcs1_der(pk_info.private_key) - .map_err(|_| type_error("invalid PKCS#1 private key"))?; + .map_err(|_| AsymmetricPrivateKeyError::InvalidPkcs1PrivateKey)?; AsymmetricPrivateKey::Rsa(private_key) } RSASSA_PSS_OID => { let details = parse_rsa_pss_params(pk_info.algorithm.parameters)?; let private_key = rsa::RsaPrivateKey::from_pkcs1_der(pk_info.private_key) - .map_err(|_| type_error("invalid PKCS#1 private key"))?; + .map_err(|_| AsymmetricPrivateKeyError::InvalidPkcs1PrivateKey)?; AsymmetricPrivateKey::RsaPss(RsaPssPrivateKey { key: private_key, details, @@ -468,40 +624,43 @@ impl KeyObjectHandle { } DSA_OID => { let private_key = dsa::SigningKey::try_from(pk_info) - .map_err(|_| type_error("invalid DSA private key"))?; + .map_err(|_| AsymmetricPrivateKeyError::InvalidDsaPrivateKey)?; AsymmetricPrivateKey::Dsa(private_key) } EC_OID => { let named_curve = pk_info.algorithm.parameters_oid().map_err(|_| { - type_error("malformed or missing named curve in ec parameters") + AsymmetricPrivateKeyError::MalformedOrMissingNamedCurveInEcParameters })?; match named_curve { ID_SECP224R1_OID => { - let secret_key = - p224::SecretKey::from_sec1_der(pk_info.private_key) - .map_err(|_| type_error("invalid SEC1 private key"))?; + let secret_key = p224::SecretKey::from_sec1_der( + pk_info.private_key, + ) + .map_err(|_| AsymmetricPrivateKeyError::InvalidSec1PrivateKey)?; AsymmetricPrivateKey::Ec(EcPrivateKey::P224(secret_key)) } ID_SECP256R1_OID => { - let secret_key = - p256::SecretKey::from_sec1_der(pk_info.private_key) - .map_err(|_| type_error("invalid SEC1 private key"))?; + let secret_key = p256::SecretKey::from_sec1_der( + pk_info.private_key, + ) + .map_err(|_| AsymmetricPrivateKeyError::InvalidSec1PrivateKey)?; AsymmetricPrivateKey::Ec(EcPrivateKey::P256(secret_key)) } ID_SECP384R1_OID => { - let secret_key = - p384::SecretKey::from_sec1_der(pk_info.private_key) - .map_err(|_| type_error("invalid SEC1 private key"))?; + let secret_key = p384::SecretKey::from_sec1_der( + pk_info.private_key, + ) + .map_err(|_| AsymmetricPrivateKeyError::InvalidSec1PrivateKey)?; AsymmetricPrivateKey::Ec(EcPrivateKey::P384(secret_key)) } - _ => return Err(type_error("unsupported ec named curve")), + _ => return Err(AsymmetricPrivateKeyError::UnsupportedEcNamedCurve), } } X25519_OID => { let string_ref = OctetStringRef::from_der(pk_info.private_key) - .map_err(|_| type_error("invalid x25519 private key"))?; + .map_err(|_| AsymmetricPrivateKeyError::InvalidX25519PrivateKey)?; if string_ref.as_bytes().len() != 32 { - return Err(type_error("x25519 private key is the wrong length")); + return Err(AsymmetricPrivateKeyError::X25519PrivateKeyIsWrongLength); } let mut bytes = [0; 32]; bytes.copy_from_slice(string_ref.as_bytes()); @@ -509,22 +668,22 @@ impl KeyObjectHandle { } ED25519_OID => { let signing_key = ed25519_dalek::SigningKey::try_from(pk_info) - .map_err(|_| type_error("invalid Ed25519 private key"))?; + .map_err(|_| AsymmetricPrivateKeyError::InvalidEd25519PrivateKey)?; AsymmetricPrivateKey::Ed25519(signing_key) } DH_KEY_AGREEMENT_OID => { let params = pk_info .algorithm .parameters - .ok_or_else(|| type_error("missing dh parameters"))?; + .ok_or(AsymmetricPrivateKeyError::MissingDhParameters)?; let params = pkcs3::DhParameter::from_der(¶ms.to_der().unwrap()) - .map_err(|_| type_error("malformed dh parameters"))?; + .map_err(|_| AsymmetricPrivateKeyError::MissingDhParameters)?; AsymmetricPrivateKey::Dh(DhPrivateKey { key: dh::PrivateKey::from_bytes(pk_info.private_key), params, }) } - _ => return Err(type_error("unsupported private key oid")), + _ => return Err(AsymmetricPrivateKeyError::UnsupportedPrivateKeyOid), }; Ok(KeyObjectHandle::AsymmetricPrivate(private_key)) @@ -532,7 +691,7 @@ impl KeyObjectHandle { pub fn new_x509_public_key( spki: &x509::SubjectPublicKeyInfo, - ) -> Result { + ) -> Result { use x509_parser::der_parser::asn1_rs::oid; use x509_parser::public_key::PublicKey; @@ -565,18 +724,18 @@ impl KeyObjectHandle { let public_key = p384::PublicKey::from_sec1_bytes(data)?; AsymmetricPublicKey::Ec(EcPublicKey::P384(public_key)) } - _ => return Err(type_error("unsupported ec named curve")), + _ => return Err(X509PublicKeyError::UnsupportedEcNamedCurve), } } else { - return Err(type_error("missing ec parameters")); + return Err(X509PublicKeyError::MissingEcParameters); } } PublicKey::DSA(_) => { let verifying_key = dsa::VerifyingKey::from_public_key_der(spki.raw) - .map_err(|_| type_error("malformed DSS public key"))?; + .map_err(|_| X509PublicKeyError::MalformedDssPublicKey)?; AsymmetricPublicKey::Dsa(verifying_key) } - _ => return Err(type_error("unsupported x509 public key type")), + _ => return Err(X509PublicKeyError::UnsupportedX509KeyType), }; Ok(KeyObjectHandle::AsymmetricPublic(key)) @@ -585,7 +744,7 @@ impl KeyObjectHandle { pub fn new_rsa_jwk( jwk: RsaJwkKey, is_public: bool, - ) -> Result { + ) -> Result { use base64::prelude::BASE64_URL_SAFE_NO_PAD; let n = BASE64_URL_SAFE_NO_PAD.decode(jwk.n.as_bytes())?; @@ -604,19 +763,19 @@ impl KeyObjectHandle { let d = BASE64_URL_SAFE_NO_PAD.decode( jwk .d - .ok_or_else(|| type_error("missing RSA private component"))? + .ok_or(RsaJwkError::MissingRsaPrivateComponent)? .as_bytes(), )?; let p = BASE64_URL_SAFE_NO_PAD.decode( jwk .p - .ok_or_else(|| type_error("missing RSA private component"))? + .ok_or(RsaJwkError::MissingRsaPrivateComponent)? .as_bytes(), )?; let q = BASE64_URL_SAFE_NO_PAD.decode( jwk .q - .ok_or_else(|| type_error("missing RSA private component"))? + .ok_or(RsaJwkError::MissingRsaPrivateComponent)? .as_bytes(), )?; @@ -640,7 +799,7 @@ impl KeyObjectHandle { pub fn new_ec_jwk( jwk: &JwkEcKey, is_public: bool, - ) -> Result { + ) -> Result { // https://datatracker.ietf.org/doc/html/rfc7518#section-6.2.1.1 let handle = match jwk.crv() { "P-256" if is_public => { @@ -660,7 +819,7 @@ impl KeyObjectHandle { EcPrivateKey::P384(p384::SecretKey::from_jwk(jwk)?), )), _ => { - return Err(type_error(format!("unsupported curve: {}", jwk.crv()))); + return Err(EcJwkError::UnsupportedCurve(jwk.crv().to_string())); } }; @@ -671,12 +830,11 @@ impl KeyObjectHandle { curve: &str, data: &[u8], is_public: bool, - ) -> Result { + ) -> Result { match curve { "Ed25519" => { - let data = data - .try_into() - .map_err(|_| type_error("invalid Ed25519 key"))?; + let data = + data.try_into().map_err(|_| EdRawError::InvalidEd25519Key)?; if !is_public { Ok(KeyObjectHandle::AsymmetricPrivate( AsymmetricPrivateKey::Ed25519( @@ -692,9 +850,8 @@ impl KeyObjectHandle { } } "X25519" => { - let data: [u8; 32] = data - .try_into() - .map_err(|_| type_error("invalid x25519 key"))?; + let data: [u8; 32] = + data.try_into().map_err(|_| EdRawError::InvalidEd25519Key)?; if !is_public { Ok(KeyObjectHandle::AsymmetricPrivate( AsymmetricPrivateKey::X25519(x25519_dalek::StaticSecret::from( @@ -707,7 +864,7 @@ impl KeyObjectHandle { )) } } - _ => Err(type_error("unsupported curve")), + _ => Err(EdRawError::UnsupportedCurve), } } @@ -716,24 +873,23 @@ impl KeyObjectHandle { format: &str, typ: &str, passphrase: Option<&[u8]>, - ) -> Result { + ) -> Result { let document = match format { "pem" => { let pem = std::str::from_utf8(key).map_err(|err| { - type_error(format!( - "invalid PEM public key: not valid utf8 starting at byte {}", - err.valid_up_to() - )) + AsymmetricPublicKeyError::InvalidPemPrivateKeyInvalidUtf8( + err.valid_up_to(), + ) })?; let (label, document) = Document::from_pem(pem) - .map_err(|_| type_error("invalid PEM public key"))?; + .map_err(|_| AsymmetricPublicKeyError::InvalidPemPublicKey)?; match label { SubjectPublicKeyInfoRef::PEM_LABEL => document, rsa::pkcs1::RsaPublicKey::PEM_LABEL => { Document::from_pkcs1_der(document.as_bytes()) - .map_err(|_| type_error("invalid PKCS#1 public key"))? + .map_err(|_| AsymmetricPublicKeyError::InvalidPkcs1PublicKey)? } EncryptedPrivateKeyInfo::PEM_LABEL | PrivateKeyInfo::PEM_LABEL @@ -754,27 +910,36 @@ impl KeyObjectHandle { } "CERTIFICATE" => { let (_, pem) = x509_parser::pem::parse_x509_pem(pem.as_bytes()) - .map_err(|_| type_error("invalid x509 certificate"))?; + .map_err(|_| AsymmetricPublicKeyError::InvalidX509Certificate)?; let cert = pem.parse_x509()?; let public_key = cert.tbs_certificate.subject_pki; - return KeyObjectHandle::new_x509_public_key(&public_key); + return KeyObjectHandle::new_x509_public_key(&public_key) + .map_err(Into::into); } _ => { - return Err(type_error(format!("unsupported PEM label: {}", label))) + return Err(AsymmetricPublicKeyError::UnsupportedPemLabel( + label.to_string(), + )) } } } "der" => match typ { "pkcs1" => Document::from_pkcs1_der(key) - .map_err(|_| type_error("invalid PKCS#1 public key"))?, + .map_err(|_| AsymmetricPublicKeyError::InvalidPkcs1PublicKey)?, "spki" => Document::from_public_key_der(key) - .map_err(|_| type_error("invalid SPKI public key"))?, - _ => return Err(type_error(format!("unsupported key type: {}", typ))), + .map_err(|_| AsymmetricPublicKeyError::InvalidSpkiPublicKey)?, + _ => { + return Err(AsymmetricPublicKeyError::UnsupportedKeyType( + typ.to_string(), + )) + } }, _ => { - return Err(type_error(format!("unsupported key format: {}", format))) + return Err(AsymmetricPublicKeyError::UnsupportedKeyType( + format.to_string(), + )) } }; @@ -799,16 +964,16 @@ impl KeyObjectHandle { } DSA_OID => { let verifying_key = dsa::VerifyingKey::try_from(spki) - .map_err(|_| type_error("malformed DSS public key"))?; + .map_err(|_| AsymmetricPublicKeyError::MalformedDssPublicKey)?; AsymmetricPublicKey::Dsa(verifying_key) } EC_OID => { let named_curve = spki.algorithm.parameters_oid().map_err(|_| { - type_error("malformed or missing named curve in ec parameters") - })?; - let data = spki.subject_public_key.as_bytes().ok_or_else(|| { - type_error("malformed or missing public key in ec spki") + AsymmetricPublicKeyError::MalformedOrMissingNamedCurveInEcParameters })?; + let data = spki.subject_public_key.as_bytes().ok_or( + AsymmetricPublicKeyError::MalformedOrMissingPublicKeyInEcSpki, + )?; match named_curve { ID_SECP224R1_OID => { @@ -823,54 +988,68 @@ impl KeyObjectHandle { let public_key = p384::PublicKey::from_sec1_bytes(data)?; AsymmetricPublicKey::Ec(EcPublicKey::P384(public_key)) } - _ => return Err(type_error("unsupported ec named curve")), + _ => return Err(AsymmetricPublicKeyError::UnsupportedEcNamedCurve), } } X25519_OID => { let mut bytes = [0; 32]; - let data = spki.subject_public_key.as_bytes().ok_or_else(|| { - type_error("malformed or missing public key in x25519 spki") - })?; + let data = spki.subject_public_key.as_bytes().ok_or( + AsymmetricPublicKeyError::MalformedOrMissingPublicKeyInX25519Spki, + )?; if data.len() < 32 { - return Err(type_error("x25519 public key is too short")); + return Err(AsymmetricPublicKeyError::X25519PublicKeyIsTooShort); } bytes.copy_from_slice(&data[0..32]); AsymmetricPublicKey::X25519(x25519_dalek::PublicKey::from(bytes)) } ED25519_OID => { let verifying_key = ed25519_dalek::VerifyingKey::try_from(spki) - .map_err(|_| type_error("invalid Ed25519 private key"))?; + .map_err(|_| AsymmetricPublicKeyError::InvalidEd25519PublicKey)?; AsymmetricPublicKey::Ed25519(verifying_key) } DH_KEY_AGREEMENT_OID => { let params = spki .algorithm .parameters - .ok_or_else(|| type_error("missing dh parameters"))?; + .ok_or(AsymmetricPublicKeyError::MissingDhParameters)?; let params = pkcs3::DhParameter::from_der(¶ms.to_der().unwrap()) - .map_err(|_| type_error("malformed dh parameters"))?; + .map_err(|_| AsymmetricPublicKeyError::MalformedDhParameters)?; let Some(subject_public_key) = spki.subject_public_key.as_bytes() else { - return Err(type_error("malformed or missing public key in dh spki")); + return Err( + AsymmetricPublicKeyError::MalformedOrMissingPublicKeyInDhSpki, + ); }; AsymmetricPublicKey::Dh(DhPublicKey { key: dh::PublicKey::from_bytes(subject_public_key), params, }) } - _ => return Err(type_error("unsupported public key oid")), + _ => return Err(AsymmetricPublicKeyError::UnsupportedPrivateKeyOid), }; Ok(KeyObjectHandle::AsymmetricPublic(public_key)) } } +#[derive(Debug, thiserror::Error)] +pub enum RsaPssParamsParseError { + #[error("malformed pss private key parameters")] + MalformedPssPrivateKeyParameters, + #[error("unsupported pss hash algorithm")] + UnsupportedPssHashAlgorithm, + #[error("unsupported pss mask gen algorithm")] + UnsupportedPssMaskGenAlgorithm, + #[error("malformed or missing pss mask gen algorithm parameters")] + MalformedOrMissingPssMaskGenAlgorithm, +} + fn parse_rsa_pss_params( parameters: Option>, -) -> Result, deno_core::anyhow::Error> { +) -> Result, RsaPssParamsParseError> { let details = if let Some(parameters) = parameters { let params = RsaPssParameters::try_from(parameters) - .map_err(|_| type_error("malformed pss private key parameters"))?; + .map_err(|_| RsaPssParamsParseError::MalformedPssPrivateKeyParameters)?; let hash_algorithm = match params.hash_algorithm.map(|k| k.oid) { Some(ID_SHA1_OID) => RsaPssHashAlgorithm::Sha1, @@ -881,16 +1060,16 @@ fn parse_rsa_pss_params( Some(ID_SHA512_224_OID) => RsaPssHashAlgorithm::Sha512_224, Some(ID_SHA512_256_OID) => RsaPssHashAlgorithm::Sha512_256, None => RsaPssHashAlgorithm::Sha1, - _ => return Err(type_error("unsupported pss hash algorithm")), + _ => return Err(RsaPssParamsParseError::UnsupportedPssHashAlgorithm), }; let mf1_hash_algorithm = match params.mask_gen_algorithm { Some(alg) => { if alg.oid != ID_MFG1 { - return Err(type_error("unsupported pss mask gen algorithm")); + return Err(RsaPssParamsParseError::UnsupportedPssMaskGenAlgorithm); } let params = alg.parameters_oid().map_err(|_| { - type_error("malformed or missing pss mask gen algorithm parameters") + RsaPssParamsParseError::MalformedOrMissingPssMaskGenAlgorithm })?; match params { ID_SHA1_OID => RsaPssHashAlgorithm::Sha1, @@ -900,7 +1079,9 @@ fn parse_rsa_pss_params( ID_SHA512_OID => RsaPssHashAlgorithm::Sha512, ID_SHA512_224_OID => RsaPssHashAlgorithm::Sha512_224, ID_SHA512_256_OID => RsaPssHashAlgorithm::Sha512_256, - _ => return Err(type_error("unsupported pss mask gen algorithm")), + _ => { + return Err(RsaPssParamsParseError::UnsupportedPssMaskGenAlgorithm) + } } } None => hash_algorithm, @@ -921,14 +1102,49 @@ fn parse_rsa_pss_params( Ok(details) } -use base64::prelude::BASE64_URL_SAFE_NO_PAD; - fn bytes_to_b64(bytes: &[u8]) -> String { + use base64::prelude::BASE64_URL_SAFE_NO_PAD; BASE64_URL_SAFE_NO_PAD.encode(bytes) } +#[derive(Debug, thiserror::Error)] +pub enum AsymmetricPublicKeyJwkError { + #[error("key is not an asymmetric public key")] + KeyIsNotAsymmetricPublicKey, + #[error("Unsupported JWK EC curve: P224")] + UnsupportedJwkEcCurveP224, + #[error("jwk export not implemented for this key type")] + JwkExportNotImplementedForKeyType, +} + +#[derive(Debug, thiserror::Error)] +pub enum AsymmetricPublicKeyDerError { + #[error("key is not an asymmetric public key")] + KeyIsNotAsymmetricPublicKey, + #[error("invalid RSA public key")] + InvalidRsaPublicKey, + #[error("exporting non-RSA public key as PKCS#1 is not supported")] + ExportingNonRsaPublicKeyAsPkcs1Unsupported, + #[error("invalid EC public key")] + InvalidEcPublicKey, + #[error("exporting RSA-PSS public key as SPKI is not supported yet")] + ExportingNonRsaPssPublicKeyAsSpkiUnsupported, + #[error("invalid DSA public key")] + InvalidDsaPublicKey, + #[error("invalid X25519 public key")] + InvalidX25519PublicKey, + #[error("invalid Ed25519 public key")] + InvalidEd25519PublicKey, + #[error("invalid DH public key")] + InvalidDhPublicKey, + #[error("unsupported key type: {0}")] + UnsupportedKeyType(String), +} + impl AsymmetricPublicKey { - fn export_jwk(&self) -> Result { + fn export_jwk( + &self, + ) -> Result { match self { AsymmetricPublicKey::Ec(key) => { let jwk = key.to_jwk()?; @@ -974,40 +1190,39 @@ impl AsymmetricPublicKey { }); Ok(jwk) } - _ => Err(type_error("jwk export not implemented for this key type")), + _ => Err(AsymmetricPublicKeyJwkError::JwkExportNotImplementedForKeyType), } } - fn export_der(&self, typ: &str) -> Result, AnyError> { + fn export_der( + &self, + typ: &str, + ) -> Result, AsymmetricPublicKeyDerError> { match typ { "pkcs1" => match self { AsymmetricPublicKey::Rsa(key) => { let der = key .to_pkcs1_der() - .map_err(|_| type_error("invalid RSA public key"))? + .map_err(|_| AsymmetricPublicKeyDerError::InvalidRsaPublicKey)? .into_vec() .into_boxed_slice(); Ok(der) } - _ => Err(type_error( - "exporting non-RSA public key as PKCS#1 is not supported", - )), + _ => Err(AsymmetricPublicKeyDerError::ExportingNonRsaPublicKeyAsPkcs1Unsupported), }, "spki" => { let der = match self { AsymmetricPublicKey::Rsa(key) => key .to_public_key_der() - .map_err(|_| type_error("invalid RSA public key"))? + .map_err(|_| AsymmetricPublicKeyDerError::InvalidRsaPublicKey)? .into_vec() .into_boxed_slice(), AsymmetricPublicKey::RsaPss(_key) => { - return Err(generic_error( - "exporting RSA-PSS public key as SPKI is not supported yet", - )) + return Err(AsymmetricPublicKeyDerError::ExportingNonRsaPssPublicKeyAsSpkiUnsupported) } AsymmetricPublicKey::Dsa(key) => key .to_public_key_der() - .map_err(|_| type_error("invalid DSA public key"))? + .map_err(|_| AsymmetricPublicKeyDerError::InvalidDsaPublicKey)? .into_vec() .into_boxed_slice(), AsymmetricPublicKey::Ec(key) => { @@ -1023,12 +1238,12 @@ impl AsymmetricPublicKey { parameters: Some(asn1::AnyRef::from(&oid)), }, subject_public_key: BitStringRef::from_bytes(&sec1) - .map_err(|_| type_error("invalid EC public key"))?, + .map_err(|_| AsymmetricPublicKeyDerError::InvalidEcPublicKey)?, }; spki .to_der() - .map_err(|_| type_error("invalid EC public key"))? + .map_err(|_| AsymmetricPublicKeyDerError::InvalidEcPublicKey)? .into_boxed_slice() } AsymmetricPublicKey::X25519(key) => { @@ -1038,12 +1253,12 @@ impl AsymmetricPublicKey { parameters: None, }, subject_public_key: BitStringRef::from_bytes(key.as_bytes()) - .map_err(|_| type_error("invalid X25519 public key"))?, + .map_err(|_| AsymmetricPublicKeyDerError::InvalidX25519PublicKey)?, }; spki .to_der() - .map_err(|_| type_error("invalid X25519 public key"))? + .map_err(|_| AsymmetricPublicKeyDerError::InvalidX25519PublicKey)? .into_boxed_slice() } AsymmetricPublicKey::Ed25519(key) => { @@ -1053,12 +1268,12 @@ impl AsymmetricPublicKey { parameters: None, }, subject_public_key: BitStringRef::from_bytes(key.as_bytes()) - .map_err(|_| type_error("invalid Ed25519 public key"))?, + .map_err(|_| AsymmetricPublicKeyDerError::InvalidEd25519PublicKey)?, }; spki .to_der() - .map_err(|_| type_error("invalid Ed25519 public key"))? + .map_err(|_| AsymmetricPublicKeyDerError::InvalidEd25519PublicKey)? .into_boxed_slice() } AsymmetricPublicKey::Dh(key) => { @@ -1071,43 +1286,67 @@ impl AsymmetricPublicKey { }, subject_public_key: BitStringRef::from_bytes(&public_key_bytes) .map_err(|_| { - type_error("invalid DH public key") + AsymmetricPublicKeyDerError::InvalidDhPublicKey })?, }; spki .to_der() - .map_err(|_| type_error("invalid DH public key"))? + .map_err(|_| AsymmetricPublicKeyDerError::InvalidDhPublicKey)? .into_boxed_slice() } }; Ok(der) } - _ => Err(type_error(format!("unsupported key type: {}", typ))), + _ => Err(AsymmetricPublicKeyDerError::UnsupportedKeyType(typ.to_string())), } } } +#[derive(Debug, thiserror::Error)] +pub enum AsymmetricPrivateKeyDerError { + #[error("key is not an asymmetric private key")] + KeyIsNotAsymmetricPrivateKey, + #[error("invalid RSA private key")] + InvalidRsaPrivateKey, + #[error("exporting non-RSA private key as PKCS#1 is not supported")] + ExportingNonRsaPrivateKeyAsPkcs1Unsupported, + #[error("invalid EC private key")] + InvalidEcPrivateKey, + #[error("exporting non-EC private key as SEC1 is not supported")] + ExportingNonEcPrivateKeyAsSec1Unsupported, + #[error("exporting RSA-PSS private key as PKCS#8 is not supported yet")] + ExportingNonRsaPssPrivateKeyAsPkcs8Unsupported, + #[error("invalid DSA private key")] + InvalidDsaPrivateKey, + #[error("invalid X25519 private key")] + InvalidX25519PrivateKey, + #[error("invalid Ed25519 private key")] + InvalidEd25519PrivateKey, + #[error("invalid DH private key")] + InvalidDhPrivateKey, + #[error("unsupported key type: {0}")] + UnsupportedKeyType(String), +} + impl AsymmetricPrivateKey { fn export_der( &self, typ: &str, // cipher: Option<&str>, // passphrase: Option<&str>, - ) -> Result, AnyError> { + ) -> Result, AsymmetricPrivateKeyDerError> { match typ { "pkcs1" => match self { AsymmetricPrivateKey::Rsa(key) => { let der = key .to_pkcs1_der() - .map_err(|_| type_error("invalid RSA private key"))? + .map_err(|_| AsymmetricPrivateKeyDerError::InvalidRsaPrivateKey)? .to_bytes() .to_vec() .into_boxed_slice(); Ok(der) } - _ => Err(type_error( - "exporting non-RSA private key as PKCS#1 is not supported", - )), + _ => Err(AsymmetricPrivateKeyDerError::ExportingNonRsaPrivateKeyAsPkcs1Unsupported), }, "sec1" => match self { AsymmetricPrivateKey::Ec(key) => { @@ -1116,30 +1355,26 @@ impl AsymmetricPrivateKey { EcPrivateKey::P256(key) => key.to_sec1_der(), EcPrivateKey::P384(key) => key.to_sec1_der(), } - .map_err(|_| type_error("invalid EC private key"))?; + .map_err(|_| AsymmetricPrivateKeyDerError::InvalidEcPrivateKey)?; Ok(sec1.to_vec().into_boxed_slice()) } - _ => Err(type_error( - "exporting non-EC private key as SEC1 is not supported", - )), + _ => Err(AsymmetricPrivateKeyDerError::ExportingNonEcPrivateKeyAsSec1Unsupported), }, "pkcs8" => { let der = match self { AsymmetricPrivateKey::Rsa(key) => { let document = key .to_pkcs8_der() - .map_err(|_| type_error("invalid RSA private key"))?; + .map_err(|_| AsymmetricPrivateKeyDerError::InvalidRsaPrivateKey)?; document.to_bytes().to_vec().into_boxed_slice() } AsymmetricPrivateKey::RsaPss(_key) => { - return Err(generic_error( - "exporting RSA-PSS private key as PKCS#8 is not supported yet", - )) + return Err(AsymmetricPrivateKeyDerError::ExportingNonRsaPssPrivateKeyAsPkcs8Unsupported) } AsymmetricPrivateKey::Dsa(key) => { let document = key .to_pkcs8_der() - .map_err(|_| type_error("invalid DSA private key"))?; + .map_err(|_| AsymmetricPrivateKeyDerError::InvalidDsaPrivateKey)?; document.to_bytes().to_vec().into_boxed_slice() } AsymmetricPrivateKey::Ec(key) => { @@ -1148,14 +1383,14 @@ impl AsymmetricPrivateKey { EcPrivateKey::P256(key) => key.to_pkcs8_der(), EcPrivateKey::P384(key) => key.to_pkcs8_der(), } - .map_err(|_| type_error("invalid EC private key"))?; + .map_err(|_| AsymmetricPrivateKeyDerError::InvalidEcPrivateKey)?; document.to_bytes().to_vec().into_boxed_slice() } AsymmetricPrivateKey::X25519(key) => { let private_key = OctetStringRef::new(key.as_bytes()) - .map_err(|_| type_error("invalid X25519 private key"))? + .map_err(|_| AsymmetricPrivateKeyDerError::InvalidX25519PrivateKey)? .to_der() - .map_err(|_| type_error("invalid X25519 private key"))?; + .map_err(|_| AsymmetricPrivateKeyDerError::InvalidX25519PrivateKey)?; let private_key = PrivateKeyInfo { algorithm: rsa::pkcs8::AlgorithmIdentifierRef { @@ -1168,15 +1403,15 @@ impl AsymmetricPrivateKey { let der = private_key .to_der() - .map_err(|_| type_error("invalid X25519 private key"))? + .map_err(|_| AsymmetricPrivateKeyDerError::InvalidX25519PrivateKey)? .into_boxed_slice(); return Ok(der); } AsymmetricPrivateKey::Ed25519(key) => { let private_key = OctetStringRef::new(key.as_bytes()) - .map_err(|_| type_error("invalid Ed25519 private key"))? + .map_err(|_| AsymmetricPrivateKeyDerError::InvalidEd25519PrivateKey)? .to_der() - .map_err(|_| type_error("invalid Ed25519 private key"))?; + .map_err(|_| AsymmetricPrivateKeyDerError::InvalidEd25519PrivateKey)?; let private_key = PrivateKeyInfo { algorithm: rsa::pkcs8::AlgorithmIdentifierRef { @@ -1189,7 +1424,7 @@ impl AsymmetricPrivateKey { private_key .to_der() - .map_err(|_| type_error("invalid ED25519 private key"))? + .map_err(|_| AsymmetricPrivateKeyDerError::InvalidEd25519PrivateKey)? .into_boxed_slice() } AsymmetricPrivateKey::Dh(key) => { @@ -1206,14 +1441,14 @@ impl AsymmetricPrivateKey { private_key .to_der() - .map_err(|_| type_error("invalid DH private key"))? + .map_err(|_| AsymmetricPrivateKeyDerError::InvalidDhPrivateKey)? .into_boxed_slice() } }; Ok(der) } - _ => Err(type_error(format!("unsupported key type: {}", typ))), + _ => Err(AsymmetricPrivateKeyDerError::UnsupportedKeyType(typ.to_string())), } } } @@ -1225,7 +1460,7 @@ pub fn op_node_create_private_key( #[string] format: &str, #[string] typ: &str, #[buffer] passphrase: Option<&[u8]>, -) -> Result { +) -> Result { KeyObjectHandle::new_asymmetric_private_key_from_js( key, format, typ, passphrase, ) @@ -1237,7 +1472,7 @@ pub fn op_node_create_ed_raw( #[string] curve: &str, #[buffer] key: &[u8], is_public: bool, -) -> Result { +) -> Result { KeyObjectHandle::new_ed_raw(curve, key, is_public) } @@ -1255,16 +1490,16 @@ pub struct RsaJwkKey { pub fn op_node_create_rsa_jwk( #[serde] jwk: RsaJwkKey, is_public: bool, -) -> Result { +) -> Result { KeyObjectHandle::new_rsa_jwk(jwk, is_public) } #[op2] #[cppgc] pub fn op_node_create_ec_jwk( - #[serde] jwk: elliptic_curve::JwkEcKey, + #[serde] jwk: JwkEcKey, is_public: bool, -) -> Result { +) -> Result { KeyObjectHandle::new_ec_jwk(&jwk, is_public) } @@ -1275,7 +1510,7 @@ pub fn op_node_create_public_key( #[string] format: &str, #[string] typ: &str, #[buffer] passphrase: Option<&[u8]>, -) -> Result { +) -> Result { KeyObjectHandle::new_asymmetric_public_key_from_js( key, format, typ, passphrase, ) @@ -1293,7 +1528,7 @@ pub fn op_node_create_secret_key( #[string] pub fn op_node_get_asymmetric_key_type( #[cppgc] handle: &KeyObjectHandle, -) -> Result<&'static str, AnyError> { +) -> Result<&'static str, deno_core::error::AnyError> { match handle { KeyObjectHandle::AsymmetricPrivate(AsymmetricPrivateKey::Rsa(_)) | KeyObjectHandle::AsymmetricPublic(AsymmetricPublicKey::Rsa(_)) => { @@ -1364,7 +1599,7 @@ pub enum AsymmetricKeyDetails { #[serde] pub fn op_node_get_asymmetric_key_details( #[cppgc] handle: &KeyObjectHandle, -) -> Result { +) -> Result { match handle { KeyObjectHandle::AsymmetricPrivate(private_key) => match private_key { AsymmetricPrivateKey::Rsa(key) => { @@ -1482,12 +1717,10 @@ pub fn op_node_get_asymmetric_key_details( #[smi] pub fn op_node_get_symmetric_key_size( #[cppgc] handle: &KeyObjectHandle, -) -> Result { +) -> Result { match handle { - KeyObjectHandle::AsymmetricPrivate(_) => { - Err(type_error("asymmetric key is not a symmetric key")) - } - KeyObjectHandle::AsymmetricPublic(_) => { + KeyObjectHandle::AsymmetricPrivate(_) + | KeyObjectHandle::AsymmetricPublic(_) => { Err(type_error("asymmetric key is not a symmetric key")) } KeyObjectHandle::Secret(key) => Ok(key.len() * 8), @@ -1592,13 +1825,17 @@ pub async fn op_node_generate_rsa_key_async( .unwrap() } +#[derive(Debug, thiserror::Error)] +#[error("digest not allowed for RSA-PSS keys{}", .0.as_ref().map(|digest| format!(": {digest}")).unwrap_or_default())] +pub struct GenerateRsaPssError(Option); + fn generate_rsa_pss( modulus_length: usize, public_exponent: usize, hash_algorithm: Option<&str>, mf1_hash_algorithm: Option<&str>, salt_length: Option, -) -> Result { +) -> Result { let key = RsaPrivateKey::new_with_exp( &mut thread_rng(), modulus_length, @@ -1617,25 +1854,19 @@ fn generate_rsa_pss( let hash_algorithm = match_fixed_digest_with_oid!( hash_algorithm, fn (algorithm: Option) { - algorithm.ok_or_else(|| type_error("digest not allowed for RSA-PSS keys: {}"))? + algorithm.ok_or(GenerateRsaPssError(None))? }, _ => { - return Err(type_error(format!( - "digest not allowed for RSA-PSS keys: {}", - hash_algorithm - ))) + return Err(GenerateRsaPssError(Some(hash_algorithm.to_string()))) } ); let mf1_hash_algorithm = match_fixed_digest_with_oid!( mf1_hash_algorithm, fn (algorithm: Option) { - algorithm.ok_or_else(|| type_error("digest not allowed for RSA-PSS keys: {}"))? + algorithm.ok_or(GenerateRsaPssError(None))? }, _ => { - return Err(type_error(format!( - "digest not allowed for RSA-PSS keys: {}", - mf1_hash_algorithm - ))) + return Err(GenerateRsaPssError(Some(mf1_hash_algorithm.to_string()))) } ); let salt_length = @@ -1663,7 +1894,7 @@ pub fn op_node_generate_rsa_pss_key( #[string] hash_algorithm: Option, // todo: Option<&str> not supproted in ops yet #[string] mf1_hash_algorithm: Option, // todo: Option<&str> not supproted in ops yet #[smi] salt_length: Option, -) -> Result { +) -> Result { generate_rsa_pss( modulus_length, public_exponent, @@ -1681,7 +1912,7 @@ pub async fn op_node_generate_rsa_pss_key_async( #[string] hash_algorithm: Option, // todo: Option<&str> not supproted in ops yet #[string] mf1_hash_algorithm: Option, // todo: Option<&str> not supproted in ops yet #[smi] salt_length: Option, -) -> Result { +) -> Result { spawn_blocking(move || { generate_rsa_pss( modulus_length, @@ -1698,7 +1929,7 @@ pub async fn op_node_generate_rsa_pss_key_async( fn dsa_generate( modulus_length: usize, divisor_length: usize, -) -> Result { +) -> Result { let mut rng = rand::thread_rng(); use dsa::Components; use dsa::KeySize; @@ -1729,7 +1960,7 @@ fn dsa_generate( pub fn op_node_generate_dsa_key( #[smi] modulus_length: usize, #[smi] divisor_length: usize, -) -> Result { +) -> Result { dsa_generate(modulus_length, divisor_length) } @@ -1738,13 +1969,15 @@ pub fn op_node_generate_dsa_key( pub async fn op_node_generate_dsa_key_async( #[smi] modulus_length: usize, #[smi] divisor_length: usize, -) -> Result { +) -> Result { spawn_blocking(move || dsa_generate(modulus_length, divisor_length)) .await .unwrap() } -fn ec_generate(named_curve: &str) -> Result { +fn ec_generate( + named_curve: &str, +) -> Result { let mut rng = rand::thread_rng(); // TODO(@littledivy): Support public key point encoding. // Default is uncompressed. @@ -1776,7 +2009,7 @@ fn ec_generate(named_curve: &str) -> Result { #[cppgc] pub fn op_node_generate_ec_key( #[string] named_curve: &str, -) -> Result { +) -> Result { ec_generate(named_curve) } @@ -1784,7 +2017,7 @@ pub fn op_node_generate_ec_key( #[cppgc] pub async fn op_node_generate_ec_key_async( #[string] named_curve: String, -) -> Result { +) -> Result { spawn_blocking(move || ec_generate(&named_curve)) .await .unwrap() @@ -1840,7 +2073,7 @@ fn u32_slice_to_u8_slice(slice: &[u32]) -> &[u8] { fn dh_group_generate( group_name: &str, -) -> Result { +) -> Result { let (dh, prime, generator) = match group_name { "modp5" => ( dh::DiffieHellman::group::(), @@ -1895,7 +2128,7 @@ fn dh_group_generate( #[cppgc] pub fn op_node_generate_dh_group_key( #[string] group_name: &str, -) -> Result { +) -> Result { dh_group_generate(group_name) } @@ -1903,7 +2136,7 @@ pub fn op_node_generate_dh_group_key( #[cppgc] pub async fn op_node_generate_dh_group_key_async( #[string] group_name: String, -) -> Result { +) -> Result { spawn_blocking(move || dh_group_generate(&group_name)) .await .unwrap() @@ -1913,7 +2146,7 @@ fn dh_generate( prime: Option<&[u8]>, prime_len: usize, generator: usize, -) -> Result { +) -> KeyObjectHandlePair { let prime = prime .map(|p| p.into()) .unwrap_or_else(|| Prime::generate(prime_len)); @@ -1923,7 +2156,7 @@ fn dh_generate( base: asn1::Int::new(generator.to_be_bytes().as_slice()).unwrap(), private_value_length: None, }; - Ok(KeyObjectHandlePair::new( + KeyObjectHandlePair::new( AsymmetricPrivateKey::Dh(DhPrivateKey { key: dh.private_key, params: params.clone(), @@ -1932,7 +2165,7 @@ fn dh_generate( key: dh.public_key, params, }), - )) + ) } #[op2] @@ -1941,7 +2174,7 @@ pub fn op_node_generate_dh_key( #[buffer] prime: Option<&[u8]>, #[smi] prime_len: usize, #[smi] generator: usize, -) -> Result { +) -> KeyObjectHandlePair { dh_generate(prime, prime_len, generator) } @@ -1951,7 +2184,7 @@ pub async fn op_node_generate_dh_key_async( #[buffer(copy)] prime: Option>, #[smi] prime_len: usize, #[smi] generator: usize, -) -> Result { +) -> KeyObjectHandlePair { spawn_blocking(move || dh_generate(prime.as_deref(), prime_len, generator)) .await .unwrap() @@ -1963,21 +2196,21 @@ pub fn op_node_dh_keys_generate_and_export( #[buffer] prime: Option<&[u8]>, #[smi] prime_len: usize, #[smi] generator: usize, -) -> Result<(ToJsBuffer, ToJsBuffer), AnyError> { +) -> (ToJsBuffer, ToJsBuffer) { let prime = prime .map(|p| p.into()) .unwrap_or_else(|| Prime::generate(prime_len)); let dh = dh::DiffieHellman::new(prime, generator); let private_key = dh.private_key.into_vec().into_boxed_slice(); let public_key = dh.public_key.into_vec().into_boxed_slice(); - Ok((private_key.into(), public_key.into())) + (private_key.into(), public_key.into()) } #[op2] #[buffer] pub fn op_node_export_secret_key( #[cppgc] handle: &KeyObjectHandle, -) -> Result, AnyError> { +) -> Result, deno_core::error::AnyError> { let key = handle .as_secret_key() .ok_or_else(|| type_error("key is not a secret key"))?; @@ -1988,7 +2221,7 @@ pub fn op_node_export_secret_key( #[string] pub fn op_node_export_secret_key_b64url( #[cppgc] handle: &KeyObjectHandle, -) -> Result { +) -> Result { let key = handle .as_secret_key() .ok_or_else(|| type_error("key is not a secret key"))?; @@ -1999,23 +2232,33 @@ pub fn op_node_export_secret_key_b64url( #[serde] pub fn op_node_export_public_key_jwk( #[cppgc] handle: &KeyObjectHandle, -) -> Result { +) -> Result { let public_key = handle .as_public_key() - .ok_or_else(|| type_error("key is not an asymmetric public key"))?; + .ok_or(AsymmetricPublicKeyJwkError::KeyIsNotAsymmetricPublicKey)?; public_key.export_jwk() } +#[derive(Debug, thiserror::Error)] +pub enum ExportPublicKeyPemError { + #[error(transparent)] + AsymmetricPublicKeyDer(#[from] AsymmetricPublicKeyDerError), + #[error("very large data")] + VeryLargeData, + #[error(transparent)] + Der(#[from] der::Error), +} + #[op2] #[string] pub fn op_node_export_public_key_pem( #[cppgc] handle: &KeyObjectHandle, #[string] typ: &str, -) -> Result { +) -> Result { let public_key = handle .as_public_key() - .ok_or_else(|| type_error("key is not an asymmetric public key"))?; + .ok_or(AsymmetricPublicKeyDerError::KeyIsNotAsymmetricPublicKey)?; let data = public_key.export_der(typ)?; let label = match typ { @@ -2024,7 +2267,9 @@ pub fn op_node_export_public_key_pem( _ => unreachable!("export_der would have errored"), }; - let mut out = vec![0; 2048]; + let pem_len = der::pem::encapsulated_len(label, LineEnding::LF, data.len()) + .map_err(|_| ExportPublicKeyPemError::VeryLargeData)?; + let mut out = vec![0; pem_len]; let mut writer = PemWriter::new(label, LineEnding::LF, &mut out)?; writer.write(&data)?; let len = writer.finish()?; @@ -2038,22 +2283,32 @@ pub fn op_node_export_public_key_pem( pub fn op_node_export_public_key_der( #[cppgc] handle: &KeyObjectHandle, #[string] typ: &str, -) -> Result, AnyError> { +) -> Result, AsymmetricPublicKeyDerError> { let public_key = handle .as_public_key() - .ok_or_else(|| type_error("key is not an asymmetric public key"))?; + .ok_or(AsymmetricPublicKeyDerError::KeyIsNotAsymmetricPublicKey)?; public_key.export_der(typ) } +#[derive(Debug, thiserror::Error)] +pub enum ExportPrivateKeyPemError { + #[error(transparent)] + AsymmetricPublicKeyDer(#[from] AsymmetricPrivateKeyDerError), + #[error("very large data")] + VeryLargeData, + #[error(transparent)] + Der(#[from] der::Error), +} + #[op2] #[string] pub fn op_node_export_private_key_pem( #[cppgc] handle: &KeyObjectHandle, #[string] typ: &str, -) -> Result { +) -> Result { let private_key = handle .as_private_key() - .ok_or_else(|| type_error("key is not an asymmetric private key"))?; + .ok_or(AsymmetricPrivateKeyDerError::KeyIsNotAsymmetricPrivateKey)?; let data = private_key.export_der(typ)?; let label = match typ { @@ -2063,7 +2318,9 @@ pub fn op_node_export_private_key_pem( _ => unreachable!("export_der would have errored"), }; - let mut out = vec![0; 2048]; + let pem_len = der::pem::encapsulated_len(label, LineEnding::LF, data.len()) + .map_err(|_| ExportPrivateKeyPemError::VeryLargeData)?; + let mut out = vec![0; pem_len]; let mut writer = PemWriter::new(label, LineEnding::LF, &mut out)?; writer.write(&data)?; let len = writer.finish()?; @@ -2077,10 +2334,10 @@ pub fn op_node_export_private_key_pem( pub fn op_node_export_private_key_der( #[cppgc] handle: &KeyObjectHandle, #[string] typ: &str, -) -> Result, AnyError> { +) -> Result, AsymmetricPrivateKeyDerError> { let private_key = handle .as_private_key() - .ok_or_else(|| type_error("key is not an asymmetric private key"))?; + .ok_or(AsymmetricPrivateKeyDerError::KeyIsNotAsymmetricPrivateKey)?; private_key.export_der(typ) } @@ -2098,7 +2355,7 @@ pub fn op_node_key_type(#[cppgc] handle: &KeyObjectHandle) -> &'static str { #[cppgc] pub fn op_node_derive_public_key_from_private_key( #[cppgc] handle: &KeyObjectHandle, -) -> Result { +) -> Result { let Some(private_key) = handle.as_private_key() else { return Err(type_error("expected private key")); }; diff --git a/ext/node/ops/crypto/mod.rs b/ext/node/ops/crypto/mod.rs index 600d315587167e..e90e820909cdad 100644 --- a/ext/node/ops/crypto/mod.rs +++ b/ext/node/ops/crypto/mod.rs @@ -1,7 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use deno_core::error::generic_error; use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::unsync::spawn_blocking; use deno_core::JsBuffer; @@ -34,14 +33,14 @@ use rsa::Pkcs1v15Encrypt; use rsa::RsaPrivateKey; use rsa::RsaPublicKey; -mod cipher; +pub mod cipher; mod dh; -mod digest; +pub mod digest; pub mod keys; mod md5_sha1; mod pkcs3; mod primes; -mod sign; +pub mod sign; pub mod x509; use self::digest::match_fixed_digest_with_eager_block_buffer; @@ -58,38 +57,31 @@ pub fn op_node_check_prime( pub fn op_node_check_prime_bytes( #[anybuffer] bytes: &[u8], #[number] checks: usize, -) -> Result { +) -> bool { let candidate = BigInt::from_bytes_be(num_bigint::Sign::Plus, bytes); - Ok(primes::is_probably_prime(&candidate, checks)) + primes::is_probably_prime(&candidate, checks) } #[op2(async)] pub async fn op_node_check_prime_async( #[bigint] num: i64, #[number] checks: usize, -) -> Result { +) -> Result { // TODO(@littledivy): use rayon for CPU-bound tasks - Ok( - spawn_blocking(move || { - primes::is_probably_prime(&BigInt::from(num), checks) - }) - .await?, - ) + spawn_blocking(move || primes::is_probably_prime(&BigInt::from(num), checks)) + .await } #[op2(async)] pub fn op_node_check_prime_bytes_async( #[anybuffer] bytes: &[u8], #[number] checks: usize, -) -> Result>, AnyError> { +) -> impl Future> { let candidate = BigInt::from_bytes_be(num_bigint::Sign::Plus, bytes); // TODO(@littledivy): use rayon for CPU-bound tasks - Ok(async move { - Ok( - spawn_blocking(move || primes::is_probably_prime(&candidate, checks)) - .await?, - ) - }) + async move { + spawn_blocking(move || primes::is_probably_prime(&candidate, checks)).await + } } #[op2] @@ -97,7 +89,7 @@ pub fn op_node_check_prime_bytes_async( pub fn op_node_create_hash( #[string] algorithm: &str, output_length: Option, -) -> Result { +) -> Result { digest::Hasher::new(algorithm, output_length.map(|l| l as usize)) } @@ -145,17 +137,31 @@ pub fn op_node_hash_digest_hex( pub fn op_node_hash_clone( #[cppgc] hasher: &digest::Hasher, output_length: Option, -) -> Result, AnyError> { +) -> Result, digest::HashError> { hasher.clone_inner(output_length.map(|l| l as usize)) } +#[derive(Debug, thiserror::Error)] +pub enum PrivateEncryptDecryptError { + #[error(transparent)] + Pkcs8(#[from] pkcs8::Error), + #[error(transparent)] + Spki(#[from] spki::Error), + #[error(transparent)] + Utf8(#[from] std::str::Utf8Error), + #[error(transparent)] + Rsa(#[from] rsa::Error), + #[error("Unknown padding")] + UnknownPadding, +} + #[op2] #[serde] pub fn op_node_private_encrypt( #[serde] key: StringOrBuffer, #[serde] msg: StringOrBuffer, #[smi] padding: u32, -) -> Result { +) -> Result { let key = RsaPrivateKey::from_pkcs8_pem((&key).try_into()?)?; let mut rng = rand::thread_rng(); @@ -172,7 +178,7 @@ pub fn op_node_private_encrypt( .encrypt(&mut rng, Oaep::new::(), &msg)? .into(), ), - _ => Err(type_error("Unknown padding")), + _ => Err(PrivateEncryptDecryptError::UnknownPadding), } } @@ -182,13 +188,13 @@ pub fn op_node_private_decrypt( #[serde] key: StringOrBuffer, #[serde] msg: StringOrBuffer, #[smi] padding: u32, -) -> Result { +) -> Result { let key = RsaPrivateKey::from_pkcs8_pem((&key).try_into()?)?; match padding { 1 => Ok(key.decrypt(Pkcs1v15Encrypt, &msg)?.into()), 4 => Ok(key.decrypt(Oaep::new::(), &msg)?.into()), - _ => Err(type_error("Unknown padding")), + _ => Err(PrivateEncryptDecryptError::UnknownPadding), } } @@ -198,7 +204,7 @@ pub fn op_node_public_encrypt( #[serde] key: StringOrBuffer, #[serde] msg: StringOrBuffer, #[smi] padding: u32, -) -> Result { +) -> Result { let key = RsaPublicKey::from_public_key_pem((&key).try_into()?)?; let mut rng = rand::thread_rng(); @@ -209,7 +215,7 @@ pub fn op_node_public_encrypt( .encrypt(&mut rng, Oaep::new::(), &msg)? .into(), ), - _ => Err(type_error("Unknown padding")), + _ => Err(PrivateEncryptDecryptError::UnknownPadding), } } @@ -220,7 +226,7 @@ pub fn op_node_create_cipheriv( #[string] algorithm: &str, #[buffer] key: &[u8], #[buffer] iv: &[u8], -) -> Result { +) -> Result { let context = cipher::CipherContext::new(algorithm, key, iv)?; Ok(state.resource_table.add(context)) } @@ -262,11 +268,14 @@ pub fn op_node_cipheriv_final( auto_pad: bool, #[buffer] input: &[u8], #[anybuffer] output: &mut [u8], -) -> Result>, AnyError> { - let context = state.resource_table.take::(rid)?; +) -> Result>, cipher::CipherContextError> { + let context = state + .resource_table + .take::(rid) + .map_err(cipher::CipherContextError::Resource)?; let context = Rc::try_unwrap(context) - .map_err(|_| type_error("Cipher context is already in use"))?; - context.r#final(auto_pad, input, output) + .map_err(|_| cipher::CipherContextError::ContextInUse)?; + context.r#final(auto_pad, input, output).map_err(Into::into) } #[op2] @@ -274,10 +283,13 @@ pub fn op_node_cipheriv_final( pub fn op_node_cipheriv_take( state: &mut OpState, #[smi] rid: u32, -) -> Result>, AnyError> { - let context = state.resource_table.take::(rid)?; +) -> Result>, cipher::CipherContextError> { + let context = state + .resource_table + .take::(rid) + .map_err(cipher::CipherContextError::Resource)?; let context = Rc::try_unwrap(context) - .map_err(|_| type_error("Cipher context is already in use"))?; + .map_err(|_| cipher::CipherContextError::ContextInUse)?; Ok(context.take_tag()) } @@ -288,7 +300,7 @@ pub fn op_node_create_decipheriv( #[string] algorithm: &str, #[buffer] key: &[u8], #[buffer] iv: &[u8], -) -> Result { +) -> Result { let context = cipher::DecipherContext::new(algorithm, key, iv)?; Ok(state.resource_table.add(context)) } @@ -326,10 +338,13 @@ pub fn op_node_decipheriv_decrypt( pub fn op_node_decipheriv_take( state: &mut OpState, #[smi] rid: u32, -) -> Result<(), AnyError> { - let context = state.resource_table.take::(rid)?; +) -> Result<(), cipher::DecipherContextError> { + let context = state + .resource_table + .take::(rid) + .map_err(cipher::DecipherContextError::Resource)?; Rc::try_unwrap(context) - .map_err(|_| type_error("Cipher context is already in use"))?; + .map_err(|_| cipher::DecipherContextError::ContextInUse)?; Ok(()) } @@ -341,11 +356,16 @@ pub fn op_node_decipheriv_final( #[buffer] input: &[u8], #[anybuffer] output: &mut [u8], #[buffer] auth_tag: &[u8], -) -> Result<(), AnyError> { - let context = state.resource_table.take::(rid)?; +) -> Result<(), cipher::DecipherContextError> { + let context = state + .resource_table + .take::(rid) + .map_err(cipher::DecipherContextError::Resource)?; let context = Rc::try_unwrap(context) - .map_err(|_| type_error("Cipher context is already in use"))?; - context.r#final(auto_pad, input, output, auth_tag) + .map_err(|_| cipher::DecipherContextError::ContextInUse)?; + context + .r#final(auto_pad, input, output, auth_tag) + .map_err(Into::into) } #[op2] @@ -356,7 +376,7 @@ pub fn op_node_sign( #[string] digest_type: &str, #[smi] pss_salt_length: Option, #[smi] dsa_signature_encoding: u32, -) -> Result, AnyError> { +) -> Result, sign::KeyObjectHandlePrehashedSignAndVerifyError> { handle.sign_prehashed( digest_type, digest, @@ -373,7 +393,7 @@ pub fn op_node_verify( #[buffer] signature: &[u8], #[smi] pss_salt_length: Option, #[smi] dsa_signature_encoding: u32, -) -> Result { +) -> Result { handle.verify_prehashed( digest_type, digest, @@ -383,13 +403,21 @@ pub fn op_node_verify( ) } +#[derive(Debug, thiserror::Error)] +pub enum Pbkdf2Error { + #[error("unsupported digest: {0}")] + UnsupportedDigest(String), + #[error(transparent)] + Join(#[from] tokio::task::JoinError), +} + fn pbkdf2_sync( password: &[u8], salt: &[u8], iterations: u32, algorithm_name: &str, derived_key: &mut [u8], -) -> Result<(), AnyError> { +) -> Result<(), Pbkdf2Error> { match_fixed_digest_with_eager_block_buffer!( algorithm_name, fn () { @@ -397,10 +425,7 @@ fn pbkdf2_sync( Ok(()) }, _ => { - Err(type_error(format!( - "unsupported digest: {}", - algorithm_name - ))) + Err(Pbkdf2Error::UnsupportedDigest(algorithm_name.to_string())) } ) } @@ -424,7 +449,7 @@ pub async fn op_node_pbkdf2_async( #[smi] iterations: u32, #[string] digest: String, #[number] keylen: usize, -) -> Result { +) -> Result { spawn_blocking(move || { let mut derived_key = vec![0; keylen]; pbkdf2_sync(&password, &salt, iterations, &digest, &mut derived_key) @@ -450,15 +475,27 @@ pub async fn op_node_fill_random_async(#[smi] len: i32) -> ToJsBuffer { .unwrap() } +#[derive(Debug, thiserror::Error)] +pub enum HkdfError { + #[error("expected secret key")] + ExpectedSecretKey, + #[error("HKDF-Expand failed")] + HkdfExpandFailed, + #[error("Unsupported digest: {0}")] + UnsupportedDigest(String), + #[error(transparent)] + Join(#[from] tokio::task::JoinError), +} + fn hkdf_sync( digest_algorithm: &str, handle: &KeyObjectHandle, salt: &[u8], info: &[u8], okm: &mut [u8], -) -> Result<(), AnyError> { +) -> Result<(), HkdfError> { let Some(ikm) = handle.as_secret_key() else { - return Err(type_error("expected secret key")); + return Err(HkdfError::ExpectedSecretKey); }; match_fixed_digest_with_eager_block_buffer!( @@ -466,10 +503,10 @@ fn hkdf_sync( fn () { let hk = Hkdf::::new(Some(salt), ikm); hk.expand(info, okm) - .map_err(|_| type_error("HKDF-Expand failed")) + .map_err(|_| HkdfError::HkdfExpandFailed) }, _ => { - Err(type_error(format!("Unsupported digest: {}", digest_algorithm))) + Err(HkdfError::UnsupportedDigest(digest_algorithm.to_string())) } ) } @@ -481,7 +518,7 @@ pub fn op_node_hkdf( #[buffer] salt: &[u8], #[buffer] info: &[u8], #[buffer] okm: &mut [u8], -) -> Result<(), AnyError> { +) -> Result<(), HkdfError> { hkdf_sync(digest_algorithm, handle, salt, info, okm) } @@ -493,7 +530,7 @@ pub async fn op_node_hkdf_async( #[buffer] salt: JsBuffer, #[buffer] info: JsBuffer, #[number] okm_len: usize, -) -> Result { +) -> Result { let handle = handle.clone(); spawn_blocking(move || { let mut okm = vec![0u8; okm_len]; @@ -509,27 +546,24 @@ pub fn op_node_dh_compute_secret( #[buffer] prime: JsBuffer, #[buffer] private_key: JsBuffer, #[buffer] their_public_key: JsBuffer, -) -> Result { +) -> ToJsBuffer { let pubkey: BigUint = BigUint::from_bytes_be(their_public_key.as_ref()); let privkey: BigUint = BigUint::from_bytes_be(private_key.as_ref()); let primei: BigUint = BigUint::from_bytes_be(prime.as_ref()); let shared_secret: BigUint = pubkey.modpow(&privkey, &primei); - Ok(shared_secret.to_bytes_be().into()) + shared_secret.to_bytes_be().into() } #[op2(fast)] -#[smi] -pub fn op_node_random_int( - #[smi] min: i32, - #[smi] max: i32, -) -> Result { +#[number] +pub fn op_node_random_int(#[number] min: i64, #[number] max: i64) -> i64 { let mut rng = rand::thread_rng(); // Uniform distribution is required to avoid Modulo Bias // https://en.wikipedia.org/wiki/Fisher–Yates_shuffle#Modulo_bias let dist = Uniform::from(min..max); - Ok(dist.sample(&mut rng)) + dist.sample(&mut rng) } #[allow(clippy::too_many_arguments)] @@ -542,7 +576,7 @@ fn scrypt( parallelization: u32, _maxmem: u32, output_buffer: &mut [u8], -) -> Result<(), AnyError> { +) -> Result<(), deno_core::error::AnyError> { // Construct Params let params = scrypt::Params::new( cost as u8, @@ -573,7 +607,7 @@ pub fn op_node_scrypt_sync( #[smi] parallelization: u32, #[smi] maxmem: u32, #[anybuffer] output_buffer: &mut [u8], -) -> Result<(), AnyError> { +) -> Result<(), deno_core::error::AnyError> { scrypt( password, salt, @@ -586,6 +620,14 @@ pub fn op_node_scrypt_sync( ) } +#[derive(Debug, thiserror::Error)] +pub enum ScryptAsyncError { + #[error(transparent)] + Join(#[from] tokio::task::JoinError), + #[error(transparent)] + Other(deno_core::error::AnyError), +} + #[op2(async)] #[serde] pub async fn op_node_scrypt_async( @@ -596,10 +638,11 @@ pub async fn op_node_scrypt_async( #[smi] block_size: u32, #[smi] parallelization: u32, #[smi] maxmem: u32, -) -> Result { +) -> Result { spawn_blocking(move || { let mut output_buffer = vec![0u8; keylen as usize]; - let res = scrypt( + + scrypt( password, salt, keylen, @@ -608,25 +651,30 @@ pub async fn op_node_scrypt_async( parallelization, maxmem, &mut output_buffer, - ); - - if res.is_ok() { - Ok(output_buffer.into()) - } else { - // TODO(lev): rethrow the error? - Err(generic_error("scrypt failure")) - } + ) + .map(|_| output_buffer.into()) + .map_err(ScryptAsyncError::Other) }) .await? } +#[derive(Debug, thiserror::Error)] +pub enum EcdhEncodePubKey { + #[error("Invalid public key")] + InvalidPublicKey, + #[error("Unsupported curve")] + UnsupportedCurve, + #[error(transparent)] + Sec1(#[from] sec1::Error), +} + #[op2] #[buffer] pub fn op_node_ecdh_encode_pubkey( #[string] curve: &str, #[buffer] pubkey: &[u8], compress: bool, -) -> Result, AnyError> { +) -> Result, EcdhEncodePubKey> { use elliptic_curve::sec1::FromEncodedPoint; match curve { @@ -639,7 +687,7 @@ pub fn op_node_ecdh_encode_pubkey( ); // CtOption does not expose its variants. if pubkey.is_none().into() { - return Err(type_error("Invalid public key")); + return Err(EcdhEncodePubKey::InvalidPublicKey); } let pubkey = pubkey.unwrap(); @@ -652,7 +700,7 @@ pub fn op_node_ecdh_encode_pubkey( ); // CtOption does not expose its variants. if pubkey.is_none().into() { - return Err(type_error("Invalid public key")); + return Err(EcdhEncodePubKey::InvalidPublicKey); } let pubkey = pubkey.unwrap(); @@ -665,7 +713,7 @@ pub fn op_node_ecdh_encode_pubkey( ); // CtOption does not expose its variants. if pubkey.is_none().into() { - return Err(type_error("Invalid public key")); + return Err(EcdhEncodePubKey::InvalidPublicKey); } let pubkey = pubkey.unwrap(); @@ -678,14 +726,14 @@ pub fn op_node_ecdh_encode_pubkey( ); // CtOption does not expose its variants. if pubkey.is_none().into() { - return Err(type_error("Invalid public key")); + return Err(EcdhEncodePubKey::InvalidPublicKey); } let pubkey = pubkey.unwrap(); Ok(pubkey.to_encoded_point(compress).as_ref().to_vec()) } - &_ => Err(type_error("Unsupported curve")), + &_ => Err(EcdhEncodePubKey::UnsupportedCurve), } } @@ -695,7 +743,7 @@ pub fn op_node_ecdh_generate_keys( #[buffer] pubbuf: &mut [u8], #[buffer] privbuf: &mut [u8], #[string] format: &str, -) -> Result<(), AnyError> { +) -> Result<(), deno_core::error::AnyError> { let mut rng = rand::thread_rng(); let compress = format == "compressed"; match curve { @@ -742,7 +790,7 @@ pub fn op_node_ecdh_compute_secret( #[buffer] this_priv: Option, #[buffer] their_pub: &mut [u8], #[buffer] secret: &mut [u8], -) -> Result<(), AnyError> { +) { match curve { "secp256k1" => { let their_public_key = @@ -760,8 +808,6 @@ pub fn op_node_ecdh_compute_secret( their_public_key.as_affine(), ); secret.copy_from_slice(shared_secret.raw_secret_bytes()); - - Ok(()) } "prime256v1" | "secp256r1" => { let their_public_key = @@ -776,8 +822,6 @@ pub fn op_node_ecdh_compute_secret( their_public_key.as_affine(), ); secret.copy_from_slice(shared_secret.raw_secret_bytes()); - - Ok(()) } "secp384r1" => { let their_public_key = @@ -792,8 +836,6 @@ pub fn op_node_ecdh_compute_secret( their_public_key.as_affine(), ); secret.copy_from_slice(shared_secret.raw_secret_bytes()); - - Ok(()) } "secp224r1" => { let their_public_key = @@ -808,8 +850,6 @@ pub fn op_node_ecdh_compute_secret( their_public_key.as_affine(), ); secret.copy_from_slice(shared_secret.raw_secret_bytes()); - - Ok(()) } &_ => todo!(), } @@ -820,7 +860,7 @@ pub fn op_node_ecdh_compute_public_key( #[string] curve: &str, #[buffer] privkey: &[u8], #[buffer] pubkey: &mut [u8], -) -> Result<(), AnyError> { +) { match curve { "secp256k1" => { let this_private_key = @@ -828,8 +868,6 @@ pub fn op_node_ecdh_compute_public_key( .expect("bad private key"); let public_key = this_private_key.public_key(); pubkey.copy_from_slice(public_key.to_sec1_bytes().as_ref()); - - Ok(()) } "prime256v1" | "secp256r1" => { let this_private_key = @@ -837,7 +875,6 @@ pub fn op_node_ecdh_compute_public_key( .expect("bad private key"); let public_key = this_private_key.public_key(); pubkey.copy_from_slice(public_key.to_sec1_bytes().as_ref()); - Ok(()) } "secp384r1" => { let this_private_key = @@ -845,7 +882,6 @@ pub fn op_node_ecdh_compute_public_key( .expect("bad private key"); let public_key = this_private_key.public_key(); pubkey.copy_from_slice(public_key.to_sec1_bytes().as_ref()); - Ok(()) } "secp224r1" => { let this_private_key = @@ -853,7 +889,6 @@ pub fn op_node_ecdh_compute_public_key( .expect("bad private key"); let public_key = this_private_key.public_key(); pubkey.copy_from_slice(public_key.to_sec1_bytes().as_ref()); - Ok(()) } &_ => todo!(), } @@ -874,8 +909,20 @@ pub fn op_node_gen_prime(#[number] size: usize) -> ToJsBuffer { #[serde] pub async fn op_node_gen_prime_async( #[number] size: usize, -) -> Result { - Ok(spawn_blocking(move || gen_prime(size)).await?) +) -> Result { + spawn_blocking(move || gen_prime(size)).await +} + +#[derive(Debug, thiserror::Error)] +pub enum DiffieHellmanError { + #[error("Expected private key")] + ExpectedPrivateKey, + #[error("Expected public key")] + ExpectedPublicKey, + #[error("DH parameters mismatch")] + DhParametersMismatch, + #[error("Unsupported key type for diffie hellman, or key type mismatch")] + UnsupportedKeyTypeForDiffieHellmanOrKeyTypeMismatch, } #[op2] @@ -883,117 +930,134 @@ pub async fn op_node_gen_prime_async( pub fn op_node_diffie_hellman( #[cppgc] private: &KeyObjectHandle, #[cppgc] public: &KeyObjectHandle, -) -> Result, AnyError> { +) -> Result, DiffieHellmanError> { let private = private .as_private_key() - .ok_or_else(|| type_error("Expected private key"))?; + .ok_or(DiffieHellmanError::ExpectedPrivateKey)?; let public = public .as_public_key() - .ok_or_else(|| type_error("Expected public key"))?; - - let res = match (private, &*public) { - ( - AsymmetricPrivateKey::Ec(EcPrivateKey::P224(private)), - AsymmetricPublicKey::Ec(EcPublicKey::P224(public)), - ) => p224::ecdh::diffie_hellman( - private.to_nonzero_scalar(), - public.as_affine(), - ) - .raw_secret_bytes() - .to_vec() - .into_boxed_slice(), - ( - AsymmetricPrivateKey::Ec(EcPrivateKey::P256(private)), - AsymmetricPublicKey::Ec(EcPublicKey::P256(public)), - ) => p256::ecdh::diffie_hellman( - private.to_nonzero_scalar(), - public.as_affine(), - ) - .raw_secret_bytes() - .to_vec() - .into_boxed_slice(), - ( - AsymmetricPrivateKey::Ec(EcPrivateKey::P384(private)), - AsymmetricPublicKey::Ec(EcPublicKey::P384(public)), - ) => p384::ecdh::diffie_hellman( - private.to_nonzero_scalar(), - public.as_affine(), - ) - .raw_secret_bytes() - .to_vec() - .into_boxed_slice(), - ( - AsymmetricPrivateKey::X25519(private), - AsymmetricPublicKey::X25519(public), - ) => private - .diffie_hellman(public) - .to_bytes() - .into_iter() - .collect(), - (AsymmetricPrivateKey::Dh(private), AsymmetricPublicKey::Dh(public)) => { - if private.params.prime != public.params.prime - || private.params.base != public.params.base - { - return Err(type_error("DH parameters mismatch")); + .ok_or(DiffieHellmanError::ExpectedPublicKey)?; + + let res = + match (private, &*public) { + ( + AsymmetricPrivateKey::Ec(EcPrivateKey::P224(private)), + AsymmetricPublicKey::Ec(EcPublicKey::P224(public)), + ) => p224::ecdh::diffie_hellman( + private.to_nonzero_scalar(), + public.as_affine(), + ) + .raw_secret_bytes() + .to_vec() + .into_boxed_slice(), + ( + AsymmetricPrivateKey::Ec(EcPrivateKey::P256(private)), + AsymmetricPublicKey::Ec(EcPublicKey::P256(public)), + ) => p256::ecdh::diffie_hellman( + private.to_nonzero_scalar(), + public.as_affine(), + ) + .raw_secret_bytes() + .to_vec() + .into_boxed_slice(), + ( + AsymmetricPrivateKey::Ec(EcPrivateKey::P384(private)), + AsymmetricPublicKey::Ec(EcPublicKey::P384(public)), + ) => p384::ecdh::diffie_hellman( + private.to_nonzero_scalar(), + public.as_affine(), + ) + .raw_secret_bytes() + .to_vec() + .into_boxed_slice(), + ( + AsymmetricPrivateKey::X25519(private), + AsymmetricPublicKey::X25519(public), + ) => private + .diffie_hellman(public) + .to_bytes() + .into_iter() + .collect(), + (AsymmetricPrivateKey::Dh(private), AsymmetricPublicKey::Dh(public)) => { + if private.params.prime != public.params.prime + || private.params.base != public.params.base + { + return Err(DiffieHellmanError::DhParametersMismatch); + } + + // OSIP - Octet-String-to-Integer primitive + let public_key = public.key.clone().into_vec(); + let pubkey = BigUint::from_bytes_be(&public_key); + + // Exponentiation (z = y^x mod p) + let prime = BigUint::from_bytes_be(private.params.prime.as_bytes()); + let private_key = private.key.clone().into_vec(); + let private_key = BigUint::from_bytes_be(&private_key); + let shared_secret = pubkey.modpow(&private_key, &prime); + + shared_secret.to_bytes_be().into() } - - // OSIP - Octet-String-to-Integer primitive - let public_key = public.key.clone().into_vec(); - let pubkey = BigUint::from_bytes_be(&public_key); - - // Exponentiation (z = y^x mod p) - let prime = BigUint::from_bytes_be(private.params.prime.as_bytes()); - let private_key = private.key.clone().into_vec(); - let private_key = BigUint::from_bytes_be(&private_key); - let shared_secret = pubkey.modpow(&private_key, &prime); - - shared_secret.to_bytes_be().into() - } - _ => { - return Err(type_error( - "Unsupported key type for diffie hellman, or key type mismatch", - )) - } - }; + _ => return Err( + DiffieHellmanError::UnsupportedKeyTypeForDiffieHellmanOrKeyTypeMismatch, + ), + }; Ok(res) } +#[derive(Debug, thiserror::Error)] +pub enum SignEd25519Error { + #[error("Expected private key")] + ExpectedPrivateKey, + #[error("Expected Ed25519 private key")] + ExpectedEd25519PrivateKey, + #[error("Invalid Ed25519 private key")] + InvalidEd25519PrivateKey, +} + #[op2(fast)] pub fn op_node_sign_ed25519( #[cppgc] key: &KeyObjectHandle, #[buffer] data: &[u8], #[buffer] signature: &mut [u8], -) -> Result<(), AnyError> { +) -> Result<(), SignEd25519Error> { let private = key .as_private_key() - .ok_or_else(|| type_error("Expected private key"))?; + .ok_or(SignEd25519Error::ExpectedPrivateKey)?; let ed25519 = match private { AsymmetricPrivateKey::Ed25519(private) => private, - _ => return Err(type_error("Expected Ed25519 private key")), + _ => return Err(SignEd25519Error::ExpectedEd25519PrivateKey), }; let pair = Ed25519KeyPair::from_seed_unchecked(ed25519.as_bytes().as_slice()) - .map_err(|_| type_error("Invalid Ed25519 private key"))?; + .map_err(|_| SignEd25519Error::InvalidEd25519PrivateKey)?; signature.copy_from_slice(pair.sign(data).as_ref()); Ok(()) } +#[derive(Debug, thiserror::Error)] +pub enum VerifyEd25519Error { + #[error("Expected public key")] + ExpectedPublicKey, + #[error("Expected Ed25519 public key")] + ExpectedEd25519PublicKey, +} + #[op2(fast)] pub fn op_node_verify_ed25519( #[cppgc] key: &KeyObjectHandle, #[buffer] data: &[u8], #[buffer] signature: &[u8], -) -> Result { +) -> Result { let public = key .as_public_key() - .ok_or_else(|| type_error("Expected public key"))?; + .ok_or(VerifyEd25519Error::ExpectedPublicKey)?; let ed25519 = match &*public { AsymmetricPublicKey::Ed25519(public) => public, - _ => return Err(type_error("Expected Ed25519 public key")), + _ => return Err(VerifyEd25519Error::ExpectedEd25519PublicKey), }; let verified = ring::signature::UnparsedPublicKey::new( diff --git a/ext/node/ops/crypto/sign.rs b/ext/node/ops/crypto/sign.rs index b7779a5d807e44..30094c07654197 100644 --- a/ext/node/ops/crypto/sign.rs +++ b/ext/node/ops/crypto/sign.rs @@ -1,7 +1,4 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::generic_error; -use deno_core::error::type_error; -use deno_core::error::AnyError; use rand::rngs::OsRng; use rsa::signature::hazmat::PrehashSigner as _; use rsa::signature::hazmat::PrehashVerifier as _; @@ -26,7 +23,7 @@ use elliptic_curve::FieldBytesSize; fn dsa_signature( encoding: u32, signature: ecdsa::Signature, -) -> Result, AnyError> +) -> Result, KeyObjectHandlePrehashedSignAndVerifyError> where MaxSize: ArrayLength, as Add>::Output: Add + ArrayLength, @@ -36,10 +33,54 @@ where 0 => Ok(signature.to_der().to_bytes().to_vec().into_boxed_slice()), // IEEE P1363 1 => Ok(signature.to_bytes().to_vec().into_boxed_slice()), - _ => Err(type_error("invalid DSA signature encoding")), + _ => Err( + KeyObjectHandlePrehashedSignAndVerifyError::InvalidDsaSignatureEncoding, + ), } } +#[derive(Debug, thiserror::Error)] +pub enum KeyObjectHandlePrehashedSignAndVerifyError { + #[error("invalid DSA signature encoding")] + InvalidDsaSignatureEncoding, + #[error("key is not a private key")] + KeyIsNotPrivate, + #[error("digest not allowed for RSA signature: {0}")] + DigestNotAllowedForRsaSignature(String), + #[error("failed to sign digest with RSA")] + FailedToSignDigestWithRsa, + #[error("digest not allowed for RSA-PSS signature: {0}")] + DigestNotAllowedForRsaPssSignature(String), + #[error("failed to sign digest with RSA-PSS")] + FailedToSignDigestWithRsaPss, + #[error("failed to sign digest with DSA")] + FailedToSignDigestWithDsa, + #[error("rsa-pss with different mf1 hash algorithm and hash algorithm is not supported")] + RsaPssHashAlgorithmUnsupported, + #[error( + "private key does not allow {actual} to be used, expected {expected}" + )] + PrivateKeyDisallowsUsage { actual: String, expected: String }, + #[error("failed to sign digest")] + FailedToSignDigest, + #[error("x25519 key cannot be used for signing")] + X25519KeyCannotBeUsedForSigning, + #[error("Ed25519 key cannot be used for prehashed signing")] + Ed25519KeyCannotBeUsedForPrehashedSigning, + #[error("DH key cannot be used for signing")] + DhKeyCannotBeUsedForSigning, + #[error("key is not a public or private key")] + KeyIsNotPublicOrPrivate, + #[error("Invalid DSA signature")] + InvalidDsaSignature, + #[error("x25519 key cannot be used for verification")] + X25519KeyCannotBeUsedForVerification, + #[error("Ed25519 key cannot be used for prehashed verification")] + Ed25519KeyCannotBeUsedForPrehashedVerification, + #[error("DH key cannot be used for verification")] + DhKeyCannotBeUsedForVerification, +} + impl KeyObjectHandle { pub fn sign_prehashed( &self, @@ -47,10 +88,10 @@ impl KeyObjectHandle { digest: &[u8], pss_salt_length: Option, dsa_signature_encoding: u32, - ) -> Result, AnyError> { + ) -> Result, KeyObjectHandlePrehashedSignAndVerifyError> { let private_key = self .as_private_key() - .ok_or_else(|| type_error("key is not a private key"))?; + .ok_or(KeyObjectHandlePrehashedSignAndVerifyError::KeyIsNotPrivate)?; match private_key { AsymmetricPrivateKey::Rsa(key) => { @@ -63,34 +104,26 @@ impl KeyObjectHandle { rsa::pkcs1v15::Pkcs1v15Sign::new::() }, _ => { - return Err(type_error(format!( - "digest not allowed for RSA signature: {}", - digest_type - ))) + return Err(KeyObjectHandlePrehashedSignAndVerifyError::DigestNotAllowedForRsaSignature(digest_type.to_string())) } ) }; let signature = signer .sign(Some(&mut OsRng), key, digest) - .map_err(|_| generic_error("failed to sign digest with RSA"))?; + .map_err(|_| KeyObjectHandlePrehashedSignAndVerifyError::FailedToSignDigestWithRsa)?; Ok(signature.into()) } AsymmetricPrivateKey::RsaPss(key) => { let mut hash_algorithm = None; let mut salt_length = None; - match &key.details { - Some(details) => { - if details.hash_algorithm != details.mf1_hash_algorithm { - return Err(type_error( - "rsa-pss with different mf1 hash algorithm and hash algorithm is not supported", - )); - } - hash_algorithm = Some(details.hash_algorithm); - salt_length = Some(details.salt_length as usize); + if let Some(details) = &key.details { + if details.hash_algorithm != details.mf1_hash_algorithm { + return Err(KeyObjectHandlePrehashedSignAndVerifyError::RsaPssHashAlgorithmUnsupported); } - None => {} - }; + hash_algorithm = Some(details.hash_algorithm); + salt_length = Some(details.salt_length as usize); + } if let Some(s) = pss_salt_length { salt_length = Some(s as usize); } @@ -99,10 +132,10 @@ impl KeyObjectHandle { fn (algorithm: Option) { if let Some(hash_algorithm) = hash_algorithm.take() { if Some(hash_algorithm) != algorithm { - return Err(type_error(format!( - "private key does not allow {} to be used, expected {}", - digest_type, hash_algorithm.as_str() - ))); + return Err(KeyObjectHandlePrehashedSignAndVerifyError::PrivateKeyDisallowsUsage { + actual: digest_type.to_string(), + expected: hash_algorithm.as_str().to_string(), + }); } } if let Some(salt_length) = salt_length { @@ -112,15 +145,12 @@ impl KeyObjectHandle { } }, _ => { - return Err(type_error(format!( - "digest not allowed for RSA-PSS signature: {}", - digest_type - ))) + return Err(KeyObjectHandlePrehashedSignAndVerifyError::DigestNotAllowedForRsaPssSignature(digest_type.to_string())); } ); let signature = pss .sign(Some(&mut OsRng), &key.key, digest) - .map_err(|_| generic_error("failed to sign digest with RSA-PSS"))?; + .map_err(|_| KeyObjectHandlePrehashedSignAndVerifyError::FailedToSignDigestWithRsaPss)?; Ok(signature.into()) } AsymmetricPrivateKey::Dsa(key) => { @@ -130,15 +160,12 @@ impl KeyObjectHandle { key.sign_prehashed_rfc6979::(digest) }, _ => { - return Err(type_error(format!( - "digest not allowed for RSA signature: {}", - digest_type - ))) + return Err(KeyObjectHandlePrehashedSignAndVerifyError::DigestNotAllowedForRsaSignature(digest_type.to_string())) } ); let signature = - res.map_err(|_| generic_error("failed to sign digest with DSA"))?; + res.map_err(|_| KeyObjectHandlePrehashedSignAndVerifyError::FailedToSignDigestWithDsa)?; Ok(signature.into()) } AsymmetricPrivateKey::Ec(key) => match key { @@ -146,7 +173,7 @@ impl KeyObjectHandle { let signing_key = p224::ecdsa::SigningKey::from(key); let signature: p224::ecdsa::Signature = signing_key .sign_prehash(digest) - .map_err(|_| type_error("failed to sign digest"))?; + .map_err(|_| KeyObjectHandlePrehashedSignAndVerifyError::FailedToSignDigest)?; dsa_signature(dsa_signature_encoding, signature) } @@ -154,7 +181,7 @@ impl KeyObjectHandle { let signing_key = p256::ecdsa::SigningKey::from(key); let signature: p256::ecdsa::Signature = signing_key .sign_prehash(digest) - .map_err(|_| type_error("failed to sign digest"))?; + .map_err(|_| KeyObjectHandlePrehashedSignAndVerifyError::FailedToSignDigest)?; dsa_signature(dsa_signature_encoding, signature) } @@ -162,19 +189,17 @@ impl KeyObjectHandle { let signing_key = p384::ecdsa::SigningKey::from(key); let signature: p384::ecdsa::Signature = signing_key .sign_prehash(digest) - .map_err(|_| type_error("failed to sign digest"))?; + .map_err(|_| KeyObjectHandlePrehashedSignAndVerifyError::FailedToSignDigest)?; dsa_signature(dsa_signature_encoding, signature) } }, AsymmetricPrivateKey::X25519(_) => { - Err(type_error("x25519 key cannot be used for signing")) + Err(KeyObjectHandlePrehashedSignAndVerifyError::X25519KeyCannotBeUsedForSigning) } - AsymmetricPrivateKey::Ed25519(_) => Err(type_error( - "Ed25519 key cannot be used for prehashed signing", - )), + AsymmetricPrivateKey::Ed25519(_) => Err(KeyObjectHandlePrehashedSignAndVerifyError::Ed25519KeyCannotBeUsedForPrehashedSigning), AsymmetricPrivateKey::Dh(_) => { - Err(type_error("DH key cannot be used for signing")) + Err(KeyObjectHandlePrehashedSignAndVerifyError::DhKeyCannotBeUsedForSigning) } } } @@ -186,10 +211,10 @@ impl KeyObjectHandle { signature: &[u8], pss_salt_length: Option, dsa_signature_encoding: u32, - ) -> Result { - let public_key = self - .as_public_key() - .ok_or_else(|| type_error("key is not a public or private key"))?; + ) -> Result { + let public_key = self.as_public_key().ok_or( + KeyObjectHandlePrehashedSignAndVerifyError::KeyIsNotPublicOrPrivate, + )?; match &*public_key { AsymmetricPublicKey::Rsa(key) => { @@ -202,10 +227,7 @@ impl KeyObjectHandle { rsa::pkcs1v15::Pkcs1v15Sign::new::() }, _ => { - return Err(type_error(format!( - "digest not allowed for RSA signature: {}", - digest_type - ))) + return Err(KeyObjectHandlePrehashedSignAndVerifyError::DigestNotAllowedForRsaSignature(digest_type.to_string())) } ) }; @@ -215,18 +237,13 @@ impl KeyObjectHandle { AsymmetricPublicKey::RsaPss(key) => { let mut hash_algorithm = None; let mut salt_length = None; - match &key.details { - Some(details) => { - if details.hash_algorithm != details.mf1_hash_algorithm { - return Err(type_error( - "rsa-pss with different mf1 hash algorithm and hash algorithm is not supported", - )); - } - hash_algorithm = Some(details.hash_algorithm); - salt_length = Some(details.salt_length as usize); + if let Some(details) = &key.details { + if details.hash_algorithm != details.mf1_hash_algorithm { + return Err(KeyObjectHandlePrehashedSignAndVerifyError::RsaPssHashAlgorithmUnsupported); } - None => {} - }; + hash_algorithm = Some(details.hash_algorithm); + salt_length = Some(details.salt_length as usize); + } if let Some(s) = pss_salt_length { salt_length = Some(s as usize); } @@ -235,10 +252,10 @@ impl KeyObjectHandle { fn (algorithm: Option) { if let Some(hash_algorithm) = hash_algorithm.take() { if Some(hash_algorithm) != algorithm { - return Err(type_error(format!( - "private key does not allow {} to be used, expected {}", - digest_type, hash_algorithm.as_str() - ))); + return Err(KeyObjectHandlePrehashedSignAndVerifyError::PrivateKeyDisallowsUsage { + actual: digest_type.to_string(), + expected: hash_algorithm.as_str().to_string(), + }); } } if let Some(salt_length) = salt_length { @@ -248,17 +265,14 @@ impl KeyObjectHandle { } }, _ => { - return Err(type_error(format!( - "digest not allowed for RSA-PSS signature: {}", - digest_type - ))) + return Err(KeyObjectHandlePrehashedSignAndVerifyError::DigestNotAllowedForRsaPssSignature(digest_type.to_string())); } ); Ok(pss.verify(&key.key, digest, signature).is_ok()) } AsymmetricPublicKey::Dsa(key) => { let signature = dsa::Signature::from_der(signature) - .map_err(|_| type_error("Invalid DSA signature"))?; + .map_err(|_| KeyObjectHandlePrehashedSignAndVerifyError::InvalidDsaSignature)?; Ok(key.verify_prehash(digest, &signature).is_ok()) } AsymmetricPublicKey::Ec(key) => match key { @@ -300,13 +314,11 @@ impl KeyObjectHandle { } }, AsymmetricPublicKey::X25519(_) => { - Err(type_error("x25519 key cannot be used for verification")) + Err(KeyObjectHandlePrehashedSignAndVerifyError::X25519KeyCannotBeUsedForVerification) } - AsymmetricPublicKey::Ed25519(_) => Err(type_error( - "Ed25519 key cannot be used for prehashed verification", - )), + AsymmetricPublicKey::Ed25519(_) => Err(KeyObjectHandlePrehashedSignAndVerifyError::Ed25519KeyCannotBeUsedForPrehashedVerification), AsymmetricPublicKey::Dh(_) => { - Err(type_error("DH key cannot be used for verification")) + Err(KeyObjectHandlePrehashedSignAndVerifyError::DhKeyCannotBeUsedForVerification) } } } diff --git a/ext/node/ops/crypto/x509.rs b/ext/node/ops/crypto/x509.rs index b44ff3a4b35562..ab8e52f703dc18 100644 --- a/ext/node/ops/crypto/x509.rs +++ b/ext/node/ops/crypto/x509.rs @@ -1,11 +1,11 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::AnyError; use deno_core::op2; use x509_parser::der_parser::asn1_rs::Any; use x509_parser::der_parser::asn1_rs::Tag; use x509_parser::der_parser::oid::Oid; +pub use x509_parser::error::X509Error; use x509_parser::extensions; use x509_parser::pem; use x509_parser::prelude::*; @@ -65,7 +65,7 @@ impl<'a> Deref for CertificateView<'a> { #[cppgc] pub fn op_node_x509_parse( #[buffer] buf: &[u8], -) -> Result { +) -> Result { let source = match pem::parse_x509_pem(buf) { Ok((_, pem)) => CertificateSources::Pem(pem), Err(_) => CertificateSources::Der(buf.to_vec().into_boxed_slice()), @@ -81,7 +81,7 @@ pub fn op_node_x509_parse( X509Certificate::from_der(buf).map(|(_, cert)| cert)? } }; - Ok::<_, AnyError>(CertificateView { cert }) + Ok::<_, X509Error>(CertificateView { cert }) }, )?; @@ -89,23 +89,23 @@ pub fn op_node_x509_parse( } #[op2(fast)] -pub fn op_node_x509_ca(#[cppgc] cert: &Certificate) -> Result { +pub fn op_node_x509_ca(#[cppgc] cert: &Certificate) -> bool { let cert = cert.inner.get().deref(); - Ok(cert.is_ca()) + cert.is_ca() } #[op2(fast)] pub fn op_node_x509_check_email( #[cppgc] cert: &Certificate, #[string] email: &str, -) -> Result { +) -> bool { let cert = cert.inner.get().deref(); let subject = cert.subject(); if subject .iter_email() .any(|e| e.as_str().unwrap_or("") == email) { - return Ok(true); + return true; } let subject_alt = cert @@ -121,62 +121,60 @@ pub fn op_node_x509_check_email( for name in &subject_alt.general_names { if let extensions::GeneralName::RFC822Name(n) = name { if *n == email { - return Ok(true); + return true; } } } } - Ok(false) + false } #[op2] #[string] -pub fn op_node_x509_fingerprint( - #[cppgc] cert: &Certificate, -) -> Result, AnyError> { - Ok(cert.fingerprint::()) +pub fn op_node_x509_fingerprint(#[cppgc] cert: &Certificate) -> Option { + cert.fingerprint::() } #[op2] #[string] pub fn op_node_x509_fingerprint256( #[cppgc] cert: &Certificate, -) -> Result, AnyError> { - Ok(cert.fingerprint::()) +) -> Option { + cert.fingerprint::() } #[op2] #[string] pub fn op_node_x509_fingerprint512( #[cppgc] cert: &Certificate, -) -> Result, AnyError> { - Ok(cert.fingerprint::()) +) -> Option { + cert.fingerprint::() } #[op2] #[string] pub fn op_node_x509_get_issuer( #[cppgc] cert: &Certificate, -) -> Result { +) -> Result { let cert = cert.inner.get().deref(); - Ok(x509name_to_string(cert.issuer(), oid_registry())?) + x509name_to_string(cert.issuer(), oid_registry()) } #[op2] #[string] pub fn op_node_x509_get_subject( #[cppgc] cert: &Certificate, -) -> Result { +) -> Result { let cert = cert.inner.get().deref(); - Ok(x509name_to_string(cert.subject(), oid_registry())?) + x509name_to_string(cert.subject(), oid_registry()) } #[op2] #[cppgc] pub fn op_node_x509_public_key( #[cppgc] cert: &Certificate, -) -> Result { +) -> Result { let cert = cert.inner.get().deref(); let public_key = &cert.tbs_certificate.subject_pki; @@ -245,37 +243,29 @@ fn x509name_to_string( #[op2] #[string] -pub fn op_node_x509_get_valid_from( - #[cppgc] cert: &Certificate, -) -> Result { +pub fn op_node_x509_get_valid_from(#[cppgc] cert: &Certificate) -> String { let cert = cert.inner.get().deref(); - Ok(cert.validity().not_before.to_string()) + cert.validity().not_before.to_string() } #[op2] #[string] -pub fn op_node_x509_get_valid_to( - #[cppgc] cert: &Certificate, -) -> Result { +pub fn op_node_x509_get_valid_to(#[cppgc] cert: &Certificate) -> String { let cert = cert.inner.get().deref(); - Ok(cert.validity().not_after.to_string()) + cert.validity().not_after.to_string() } #[op2] #[string] -pub fn op_node_x509_get_serial_number( - #[cppgc] cert: &Certificate, -) -> Result { +pub fn op_node_x509_get_serial_number(#[cppgc] cert: &Certificate) -> String { let cert = cert.inner.get().deref(); let mut s = cert.serial.to_str_radix(16); s.make_ascii_uppercase(); - Ok(s) + s } #[op2(fast)] -pub fn op_node_x509_key_usage( - #[cppgc] cert: &Certificate, -) -> Result { +pub fn op_node_x509_key_usage(#[cppgc] cert: &Certificate) -> u16 { let cert = cert.inner.get().deref(); let key_usage = cert .extensions() @@ -286,5 +276,5 @@ pub fn op_node_x509_key_usage( _ => None, }); - Ok(key_usage.map(|k| k.flags).unwrap_or(0)) + key_usage.map(|k| k.flags).unwrap_or(0) } diff --git a/ext/node/ops/fs.rs b/ext/node/ops/fs.rs index 6253f32d0537f1..58a688a1fec2f1 100644 --- a/ext/node/ops/fs.rs +++ b/ext/node/ops/fs.rs @@ -3,7 +3,6 @@ use std::cell::RefCell; use std::rc::Rc; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; use deno_fs::FileSystemRc; @@ -11,11 +10,27 @@ use serde::Serialize; use crate::NodePermissions; -#[op2(fast)] +#[derive(Debug, thiserror::Error)] +pub enum FsError { + #[error(transparent)] + Permission(#[from] deno_permissions::PermissionCheckError), + #[error("{0}")] + Io(#[from] std::io::Error), + #[cfg(windows)] + #[error("Path has no root.")] + PathHasNoRoot, + #[cfg(not(any(unix, windows)))] + #[error("Unsupported platform.")] + UnsupportedPlatform, + #[error(transparent)] + Fs(#[from] deno_io::fs::FsError), +} + +#[op2(fast, stack_trace)] pub fn op_node_fs_exists_sync

( state: &mut OpState, #[string] path: String, -) -> Result +) -> Result where P: NodePermissions + 'static, { @@ -26,11 +41,11 @@ where Ok(fs.exists_sync(&path)) } -#[op2(async)] +#[op2(async, stack_trace)] pub async fn op_node_fs_exists

( state: Rc>, #[string] path: String, -) -> Result +) -> Result where P: NodePermissions + 'static, { @@ -45,12 +60,12 @@ where Ok(fs.exists_async(path).await?) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_node_cp_sync

( state: &mut OpState, #[string] path: &str, #[string] new_path: &str, -) -> Result<(), AnyError> +) -> Result<(), FsError> where P: NodePermissions + 'static, { @@ -66,12 +81,12 @@ where Ok(()) } -#[op2(async)] +#[op2(async, stack_trace)] pub async fn op_node_cp

( state: Rc>, #[string] path: String, #[string] new_path: String, -) -> Result<(), AnyError> +) -> Result<(), FsError> where P: NodePermissions + 'static, { @@ -102,13 +117,13 @@ pub struct StatFs { pub ffree: u64, } -#[op2] +#[op2(stack_trace)] #[serde] pub fn op_node_statfs

( state: Rc>, #[string] path: String, bigint: bool, -) -> Result +) -> Result where P: NodePermissions + 'static, { @@ -130,13 +145,21 @@ where let mut cpath = path.as_bytes().to_vec(); cpath.push(0); if bigint { - #[cfg(not(target_os = "macos"))] + #[cfg(not(any( + target_os = "macos", + target_os = "freebsd", + target_os = "openbsd" + )))] // SAFETY: `cpath` is NUL-terminated and result is pointer to valid statfs memory. let (code, result) = unsafe { let mut result: libc::statfs64 = std::mem::zeroed(); (libc::statfs64(cpath.as_ptr() as _, &mut result), result) }; - #[cfg(target_os = "macos")] + #[cfg(any( + target_os = "macos", + target_os = "freebsd", + target_os = "openbsd" + ))] // SAFETY: `cpath` is NUL-terminated and result is pointer to valid statfs memory. let (code, result) = unsafe { let mut result: libc::statfs = std::mem::zeroed(); @@ -146,7 +169,10 @@ where return Err(std::io::Error::last_os_error().into()); } Ok(StatFs { + #[cfg(not(target_os = "openbsd"))] typ: result.f_type as _, + #[cfg(target_os = "openbsd")] + typ: 0 as _, bsize: result.f_bsize as _, blocks: result.f_blocks as _, bfree: result.f_bfree as _, @@ -164,7 +190,10 @@ where return Err(std::io::Error::last_os_error().into()); } Ok(StatFs { + #[cfg(not(target_os = "openbsd"))] typ: result.f_type as _, + #[cfg(target_os = "openbsd")] + typ: 0 as _, bsize: result.f_bsize as _, blocks: result.f_blocks as _, bfree: result.f_bfree as _, @@ -176,7 +205,6 @@ where } #[cfg(windows)] { - use deno_core::anyhow::anyhow; use std::ffi::OsStr; use std::os::windows::ffi::OsStrExt; use windows_sys::Win32::Storage::FileSystem::GetDiskFreeSpaceW; @@ -186,10 +214,7 @@ where // call below. #[allow(clippy::disallowed_methods)] let path = path.canonicalize()?; - let root = path - .ancestors() - .last() - .ok_or(anyhow!("Path has no root."))?; + let root = path.ancestors().last().ok_or(FsError::PathHasNoRoot)?; let mut root = OsStr::new(root).encode_wide().collect::>(); root.push(0); let mut sectors_per_cluster = 0; @@ -229,11 +254,11 @@ where { let _ = path; let _ = bigint; - Err(anyhow!("Unsupported platform.")) + Err(FsError::UnsupportedPlatform) } } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_node_lutimes_sync

( state: &mut OpState, #[string] path: &str, @@ -241,7 +266,7 @@ pub fn op_node_lutimes_sync

( #[smi] atime_nanos: u32, #[number] mtime_secs: i64, #[smi] mtime_nanos: u32, -) -> Result<(), AnyError> +) -> Result<(), FsError> where P: NodePermissions + 'static, { @@ -254,7 +279,7 @@ where Ok(()) } -#[op2(async)] +#[op2(async, stack_trace)] pub async fn op_node_lutimes

( state: Rc>, #[string] path: String, @@ -262,7 +287,7 @@ pub async fn op_node_lutimes

( #[smi] atime_nanos: u32, #[number] mtime_secs: i64, #[smi] mtime_nanos: u32, -) -> Result<(), AnyError> +) -> Result<(), FsError> where P: NodePermissions + 'static, { @@ -280,13 +305,13 @@ where Ok(()) } -#[op2] +#[op2(stack_trace)] pub fn op_node_lchown_sync

( state: &mut OpState, #[string] path: String, uid: Option, gid: Option, -) -> Result<(), AnyError> +) -> Result<(), FsError> where P: NodePermissions + 'static, { @@ -298,13 +323,13 @@ where Ok(()) } -#[op2(async)] +#[op2(async, stack_trace)] pub async fn op_node_lchown

( state: Rc>, #[string] path: String, uid: Option, gid: Option, -) -> Result<(), AnyError> +) -> Result<(), FsError> where P: NodePermissions + 'static, { diff --git a/ext/node/ops/http.rs b/ext/node/ops/http.rs index 773902dedfdc28..f4adb940606651 100644 --- a/ext/node/ops/http.rs +++ b/ext/node/ops/http.rs @@ -8,14 +8,12 @@ use std::task::Context; use std::task::Poll; use bytes::Bytes; -use deno_core::anyhow; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::futures::stream::Peekable; use deno_core::futures::Future; use deno_core::futures::FutureExt; use deno_core::futures::Stream; use deno_core::futures::StreamExt; +use deno_core::futures::TryFutureExt; use deno_core::op2; use deno_core::serde::Serialize; use deno_core::unsync::spawn; @@ -33,6 +31,7 @@ use deno_core::Resource; use deno_core::ResourceId; use deno_fetch::get_or_create_client_from_state; use deno_fetch::FetchCancelHandle; +use deno_fetch::FetchError; use deno_fetch::FetchRequestResource; use deno_fetch::FetchReturn; use deno_fetch::HttpClientResource; @@ -50,7 +49,7 @@ use std::cmp::min; use tokio::io::AsyncReadExt; use tokio::io::AsyncWriteExt; -#[op2] +#[op2(stack_trace)] #[serde] pub fn op_node_http_request

( state: &mut OpState, @@ -59,12 +58,15 @@ pub fn op_node_http_request

( #[serde] headers: Vec<(ByteString, ByteString)>, #[smi] client_rid: Option, #[smi] body: Option, -) -> Result +) -> Result where P: crate::NodePermissions + 'static, { let client = if let Some(rid) = client_rid { - let r = state.resource_table.get::(rid)?; + let r = state + .resource_table + .get::(rid) + .map_err(FetchError::Resource)?; r.client.clone() } else { get_or_create_client_from_state(state)? @@ -81,10 +83,8 @@ where let mut header_map = HeaderMap::new(); for (key, value) in headers { - let name = HeaderName::from_bytes(&key) - .map_err(|err| type_error(err.to_string()))?; - let v = HeaderValue::from_bytes(&value) - .map_err(|err| type_error(err.to_string()))?; + let name = HeaderName::from_bytes(&key)?; + let v = HeaderValue::from_bytes(&value)?; header_map.append(name, v); } @@ -92,7 +92,10 @@ where let (body, con_len) = if let Some(body) = body { ( BodyExt::boxed(NodeHttpResourceToBodyAdapter::new( - state.resource_table.take_any(body)?, + state + .resource_table + .take_any(body) + .map_err(FetchError::Resource)?, )), None, ) @@ -117,7 +120,7 @@ where *request.uri_mut() = url .as_str() .parse() - .map_err(|_| type_error("Invalid URL"))?; + .map_err(|_| FetchError::InvalidUrl(url.clone()))?; *request.headers_mut() = header_map; if let Some((username, password)) = maybe_authority { @@ -136,9 +139,9 @@ where let fut = async move { client .send(request) + .map_err(Into::into) .or_cancel(cancel_handle_) .await - .map(|res| res.map_err(|err| type_error(err.to_string()))) }; let request_rid = state.resource_table.add(FetchRequestResource { @@ -174,11 +177,12 @@ pub struct NodeHttpFetchResponse { pub async fn op_node_http_fetch_send( state: Rc>, #[smi] rid: ResourceId, -) -> Result { +) -> Result { let request = state .borrow_mut() .resource_table - .take::(rid)?; + .take::(rid) + .map_err(FetchError::Resource)?; let request = Rc::try_unwrap(request) .ok() @@ -191,22 +195,23 @@ pub async fn op_node_http_fetch_send( // If any error in the chain is a hyper body error, return that as a special result we can use to // reconstruct an error chain (eg: `new TypeError(..., { cause: new Error(...) })`). // TODO(mmastrac): it would be a lot easier if we just passed a v8::Global through here instead - let mut err_ref: &dyn std::error::Error = err.as_ref(); - while let Some(err) = std::error::Error::source(err_ref) { - if let Some(err) = err.downcast_ref::() { - if let Some(err) = std::error::Error::source(err) { - return Ok(NodeHttpFetchResponse { - error: Some(err.to_string()), - ..Default::default() - }); + + if let FetchError::ClientSend(err_src) = &err { + if let Some(client_err) = std::error::Error::source(&err_src.source) { + if let Some(err_src) = client_err.downcast_ref::() { + if let Some(err_src) = std::error::Error::source(err_src) { + return Ok(NodeHttpFetchResponse { + error: Some(err_src.to_string()), + ..Default::default() + }); + } } } - err_ref = err; } - return Err(type_error(err.to_string())); + return Err(err); } - Err(_) => return Err(type_error("request was cancelled")), + Err(_) => return Err(FetchError::RequestCanceled), }; let status = res.status(); @@ -250,11 +255,12 @@ pub async fn op_node_http_fetch_send( pub async fn op_node_http_fetch_response_upgrade( state: Rc>, #[smi] rid: ResourceId, -) -> Result { +) -> Result { let raw_response = state .borrow_mut() .resource_table - .take::(rid)?; + .take::(rid) + .map_err(FetchError::Resource)?; let raw_response = Rc::try_unwrap(raw_response) .expect("Someone is holding onto NodeHttpFetchResponseResource"); @@ -277,7 +283,7 @@ pub async fn op_node_http_fetch_response_upgrade( } read_tx.write_all(&buf[..read]).await?; } - Ok::<_, AnyError>(()) + Ok::<_, FetchError>(()) }); spawn(async move { let mut buf = [0; 1024]; @@ -288,7 +294,7 @@ pub async fn op_node_http_fetch_response_upgrade( } upgraded_tx.write_all(&buf[..read]).await?; } - Ok::<_, AnyError>(()) + Ok::<_, FetchError>(()) }); } @@ -318,23 +324,26 @@ impl UpgradeStream { } } - async fn read(self: Rc, buf: &mut [u8]) -> Result { + async fn read( + self: Rc, + buf: &mut [u8], + ) -> Result { let cancel_handle = RcRef::map(self.clone(), |this| &this.cancel_handle); async { let read = RcRef::map(self, |this| &this.read); let mut read = read.borrow_mut().await; - Ok(Pin::new(&mut *read).read(buf).await?) + Pin::new(&mut *read).read(buf).await } .try_or_cancel(cancel_handle) .await } - async fn write(self: Rc, buf: &[u8]) -> Result { + async fn write(self: Rc, buf: &[u8]) -> Result { let cancel_handle = RcRef::map(self.clone(), |this| &this.cancel_handle); async { let write = RcRef::map(self, |this| &this.write); let mut write = write.borrow_mut().await; - Ok(Pin::new(&mut *write).write(buf).await?) + Pin::new(&mut *write).write(buf).await } .try_or_cancel(cancel_handle) .await @@ -387,7 +396,7 @@ impl NodeHttpFetchResponseResource { } } - pub async fn upgrade(self) -> Result { + pub async fn upgrade(self) -> Result { let reader = self.response_reader.into_inner(); match reader { NodeHttpFetchResponseReader::Start(resp) => { @@ -445,7 +454,9 @@ impl Resource for NodeHttpFetchResponseResource { // safely call `await` on it without creating a race condition. Some(_) => match reader.as_mut().next().await.unwrap() { Ok(chunk) => assert!(chunk.is_empty()), - Err(err) => break Err(type_error(err.to_string())), + Err(err) => { + break Err(deno_core::error::type_error(err.to_string())) + } }, None => break Ok(BufView::empty()), } @@ -453,7 +464,7 @@ impl Resource for NodeHttpFetchResponseResource { }; let cancel_handle = RcRef::map(self, |r| &r.cancel); - fut.try_or_cancel(cancel_handle).await + fut.try_or_cancel(cancel_handle).await.map_err(Into::into) }) } @@ -469,7 +480,9 @@ impl Resource for NodeHttpFetchResponseResource { #[allow(clippy::type_complexity)] pub struct NodeHttpResourceToBodyAdapter( Rc, - Option>>>>, + Option< + Pin>>>, + >, ); impl NodeHttpResourceToBodyAdapter { @@ -485,7 +498,7 @@ unsafe impl Send for NodeHttpResourceToBodyAdapter {} unsafe impl Sync for NodeHttpResourceToBodyAdapter {} impl Stream for NodeHttpResourceToBodyAdapter { - type Item = Result; + type Item = Result; fn poll_next( self: Pin<&mut Self>, @@ -515,7 +528,7 @@ impl Stream for NodeHttpResourceToBodyAdapter { impl hyper::body::Body for NodeHttpResourceToBodyAdapter { type Data = Bytes; - type Error = anyhow::Error; + type Error = deno_core::anyhow::Error; fn poll_frame( self: Pin<&mut Self>, diff --git a/ext/node/ops/http2.rs b/ext/node/ops/http2.rs index 9595cb33d6abfc..53dada9f4119c9 100644 --- a/ext/node/ops/http2.rs +++ b/ext/node/ops/http2.rs @@ -7,7 +7,6 @@ use std::rc::Rc; use std::task::Poll; use bytes::Bytes; -use deno_core::error::AnyError; use deno_core::futures::future::poll_fn; use deno_core::op2; use deno_core::serde::Serialize; @@ -110,17 +109,28 @@ impl Resource for Http2ServerSendResponse { } } +#[derive(Debug, thiserror::Error)] +pub enum Http2Error { + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error(transparent)] + UrlParse(#[from] url::ParseError), + #[error(transparent)] + H2(#[from] h2::Error), +} + #[op2(async)] #[serde] pub async fn op_http2_connect( state: Rc>, #[smi] rid: ResourceId, #[string] url: String, -) -> Result<(ResourceId, ResourceId), AnyError> { +) -> Result<(ResourceId, ResourceId), Http2Error> { // No permission check necessary because we're using an existing connection let network_stream = { let mut state = state.borrow_mut(); - take_network_stream_resource(&mut state.resource_table, rid)? + take_network_stream_resource(&mut state.resource_table, rid) + .map_err(Http2Error::Resource)? }; let url = Url::parse(&url)?; @@ -144,9 +154,10 @@ pub async fn op_http2_connect( pub async fn op_http2_listen( state: Rc>, #[smi] rid: ResourceId, -) -> Result { +) -> Result { let stream = - take_network_stream_resource(&mut state.borrow_mut().resource_table, rid)?; + take_network_stream_resource(&mut state.borrow_mut().resource_table, rid) + .map_err(Http2Error::Resource)?; let conn = h2::server::Builder::new().handshake(stream).await?; Ok( @@ -166,12 +177,13 @@ pub async fn op_http2_accept( #[smi] rid: ResourceId, ) -> Result< Option<(Vec<(ByteString, ByteString)>, ResourceId, ResourceId)>, - AnyError, + Http2Error, > { let resource = state .borrow() .resource_table - .get::(rid)?; + .get::(rid) + .map_err(Http2Error::Resource)?; let mut conn = RcRef::map(&resource, |r| &r.conn).borrow_mut().await; if let Some(res) = conn.accept().await { let (req, resp) = res?; @@ -233,11 +245,12 @@ pub async fn op_http2_send_response( #[smi] rid: ResourceId, #[smi] status: u16, #[serde] headers: Vec<(ByteString, ByteString)>, -) -> Result<(ResourceId, u32), AnyError> { +) -> Result<(ResourceId, u32), Http2Error> { let resource = state .borrow() .resource_table - .get::(rid)?; + .get::(rid) + .map_err(Http2Error::Resource)?; let mut send_response = RcRef::map(resource, |r| &r.send_response) .borrow_mut() .await; @@ -262,8 +275,12 @@ pub async fn op_http2_send_response( pub async fn op_http2_poll_client_connection( state: Rc>, #[smi] rid: ResourceId, -) -> Result<(), AnyError> { - let resource = state.borrow().resource_table.get::(rid)?; +) -> Result<(), Http2Error> { + let resource = state + .borrow() + .resource_table + .get::(rid) + .map_err(Http2Error::Resource)?; let cancel_handle = RcRef::map(resource.clone(), |this| &this.cancel_handle); let mut conn = RcRef::map(resource, |this| &this.conn).borrow_mut().await; @@ -289,11 +306,12 @@ pub async fn op_http2_client_request( // 4 strings of keys? #[serde] mut pseudo_headers: HashMap, #[serde] headers: Vec<(ByteString, ByteString)>, -) -> Result<(ResourceId, u32), AnyError> { +) -> Result<(ResourceId, u32), Http2Error> { let resource = state .borrow() .resource_table - .get::(client_rid)?; + .get::(client_rid) + .map_err(Http2Error::Resource)?; let url = resource.url.clone(); @@ -326,7 +344,10 @@ pub async fn op_http2_client_request( let resource = { let state = state.borrow(); - state.resource_table.get::(client_rid)? + state + .resource_table + .get::(client_rid) + .map_err(Http2Error::Resource)? }; let mut client = RcRef::map(&resource, |r| &r.client).borrow_mut().await; poll_fn(|cx| client.poll_ready(cx)).await?; @@ -345,11 +366,12 @@ pub async fn op_http2_client_send_data( #[smi] stream_rid: ResourceId, #[buffer] data: JsBuffer, end_of_stream: bool, -) -> Result<(), AnyError> { +) -> Result<(), Http2Error> { let resource = state .borrow() .resource_table - .get::(stream_rid)?; + .get::(stream_rid) + .map_err(Http2Error::Resource)?; let mut stream = RcRef::map(&resource, |r| &r.stream).borrow_mut().await; stream.send_data(data.to_vec().into(), end_of_stream)?; @@ -361,7 +383,7 @@ pub async fn op_http2_client_reset_stream( state: Rc>, #[smi] stream_rid: ResourceId, #[smi] code: u32, -) -> Result<(), AnyError> { +) -> Result<(), deno_core::error::AnyError> { let resource = state .borrow() .resource_table @@ -376,11 +398,12 @@ pub async fn op_http2_client_send_trailers( state: Rc>, #[smi] stream_rid: ResourceId, #[serde] trailers: Vec<(ByteString, ByteString)>, -) -> Result<(), AnyError> { +) -> Result<(), Http2Error> { let resource = state .borrow() .resource_table - .get::(stream_rid)?; + .get::(stream_rid) + .map_err(Http2Error::Resource)?; let mut stream = RcRef::map(&resource, |r| &r.stream).borrow_mut().await; let mut trailers_map = http::HeaderMap::new(); @@ -408,11 +431,12 @@ pub struct Http2ClientResponse { pub async fn op_http2_client_get_response( state: Rc>, #[smi] stream_rid: ResourceId, -) -> Result<(Http2ClientResponse, bool), AnyError> { +) -> Result<(Http2ClientResponse, bool), Http2Error> { let resource = state .borrow() .resource_table - .get::(stream_rid)?; + .get::(stream_rid) + .map_err(Http2Error::Resource)?; let mut response_future = RcRef::map(&resource, |r| &r.response).borrow_mut().await; @@ -478,23 +502,22 @@ fn poll_data_or_trailers( pub async fn op_http2_client_get_response_body_chunk( state: Rc>, #[smi] body_rid: ResourceId, -) -> Result<(Option>, bool, bool), AnyError> { +) -> Result<(Option>, bool, bool), Http2Error> { let resource = state .borrow() .resource_table - .get::(body_rid)?; + .get::(body_rid) + .map_err(Http2Error::Resource)?; let mut body = RcRef::map(&resource, |r| &r.body).borrow_mut().await; loop { let result = poll_fn(|cx| poll_data_or_trailers(cx, &mut body)).await; if let Err(err) = result { - let reason = err.reason(); - if let Some(reason) = reason { - if reason == Reason::CANCEL { - return Ok((None, false, true)); - } + match err.reason() { + Some(Reason::NO_ERROR) => return Ok((None, true, false)), + Some(Reason::CANCEL) => return Ok((None, false, true)), + _ => return Err(err.into()), } - return Err(err.into()); } match result.unwrap() { DataOrTrailers::Data(data) => { @@ -527,7 +550,7 @@ pub async fn op_http2_client_get_response_body_chunk( pub async fn op_http2_client_get_response_trailers( state: Rc>, #[smi] body_rid: ResourceId, -) -> Result>, AnyError> { +) -> Result>, deno_core::error::AnyError> { let resource = state .borrow() .resource_table diff --git a/ext/node/ops/idna.rs b/ext/node/ops/idna.rs index 9c9450c70ffbcc..a3d85e77c2b6b5 100644 --- a/ext/node/ops/idna.rs +++ b/ext/node/ops/idna.rs @@ -1,7 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::anyhow::Error; -use deno_core::error::range_error; use deno_core::op2; use std::borrow::Cow; @@ -11,19 +9,21 @@ use std::borrow::Cow; const PUNY_PREFIX: &str = "xn--"; -fn invalid_input_err() -> Error { - range_error("Invalid input") -} - -fn not_basic_err() -> Error { - range_error("Illegal input >= 0x80 (not a basic code point)") +#[derive(Debug, thiserror::Error)] +pub enum IdnaError { + #[error("Invalid input")] + InvalidInput, + #[error("Input would take more than 63 characters to encode")] + InputTooLong, + #[error("Illegal input >= 0x80 (not a basic code point)")] + IllegalInput, } /// map a domain by mapping each label with the given function -fn map_domain( +fn map_domain( domain: &str, - f: impl Fn(&str) -> Result, E>, -) -> Result { + f: impl Fn(&str) -> Result, IdnaError>, +) -> Result { let mut result = String::with_capacity(domain.len()); let mut domain = domain; @@ -48,7 +48,7 @@ fn map_domain( /// Maps a unicode domain to ascii by punycode encoding each label /// /// Note this is not IDNA2003 or IDNA2008 compliant, rather it matches node.js's punycode implementation -fn to_ascii(input: &str) -> Result { +fn to_ascii(input: &str) -> Result { if input.is_ascii() { return Ok(input.into()); } @@ -61,9 +61,7 @@ fn to_ascii(input: &str) -> Result { } else { idna::punycode::encode_str(label) .map(|encoded| [PUNY_PREFIX, &encoded].join("").into()) // add the prefix - .ok_or_else(|| { - Error::msg("Input would take more than 63 characters to encode") // only error possible per the docs - }) + .ok_or(IdnaError::InputTooLong) // only error possible per the docs } })?; @@ -74,13 +72,13 @@ fn to_ascii(input: &str) -> Result { /// Maps an ascii domain to unicode by punycode decoding each label /// /// Note this is not IDNA2003 or IDNA2008 compliant, rather it matches node.js's punycode implementation -fn to_unicode(input: &str) -> Result { +fn to_unicode(input: &str) -> Result { map_domain(input, |s| { if let Some(puny) = s.strip_prefix(PUNY_PREFIX) { // it's a punycode encoded label Ok( idna::punycode::decode_to_string(&puny.to_lowercase()) - .ok_or_else(invalid_input_err)? + .ok_or(IdnaError::InvalidInput)? .into(), ) } else { @@ -95,7 +93,7 @@ fn to_unicode(input: &str) -> Result { #[string] pub fn op_node_idna_punycode_to_ascii( #[string] domain: String, -) -> Result { +) -> Result { to_ascii(&domain) } @@ -105,7 +103,7 @@ pub fn op_node_idna_punycode_to_ascii( #[string] pub fn op_node_idna_punycode_to_unicode( #[string] domain: String, -) -> Result { +) -> Result { to_unicode(&domain) } @@ -115,8 +113,8 @@ pub fn op_node_idna_punycode_to_unicode( #[string] pub fn op_node_idna_domain_to_ascii( #[string] domain: String, -) -> Result { - idna::domain_to_ascii(&domain).map_err(|e| e.into()) +) -> Result { + idna::domain_to_ascii(&domain) } /// Converts a domain to Unicode as per the IDNA spec @@ -131,7 +129,7 @@ pub fn op_node_idna_domain_to_unicode(#[string] domain: String) -> String { #[string] pub fn op_node_idna_punycode_decode( #[string] domain: String, -) -> Result { +) -> Result { if domain.is_empty() { return Ok(domain); } @@ -147,11 +145,10 @@ pub fn op_node_idna_punycode_decode( .unwrap_or(domain.len() - 1); if !domain[..last_dash].is_ascii() { - return Err(not_basic_err()); + return Err(IdnaError::IllegalInput); } - idna::punycode::decode_to_string(&domain) - .ok_or_else(|| deno_core::error::range_error("Invalid input")) + idna::punycode::decode_to_string(&domain).ok_or(IdnaError::InvalidInput) } #[op2] diff --git a/ext/node/ops/inspector.rs b/ext/node/ops/inspector.rs new file mode 100644 index 00000000000000..9986aeb1970bd7 --- /dev/null +++ b/ext/node/ops/inspector.rs @@ -0,0 +1,161 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use crate::NodePermissions; +use deno_core::anyhow::Error; +use deno_core::error::generic_error; +use deno_core::futures::channel::mpsc; +use deno_core::op2; +use deno_core::v8; +use deno_core::GarbageCollected; +use deno_core::InspectorSessionKind; +use deno_core::InspectorSessionOptions; +use deno_core::JsRuntimeInspector; +use deno_core::OpState; +use std::cell::RefCell; +use std::rc::Rc; + +#[op2(fast)] +pub fn op_inspector_enabled() -> bool { + // TODO: hook up to InspectorServer + false +} + +#[op2(stack_trace)] +pub fn op_inspector_open

( + _state: &mut OpState, + _port: Option, + #[string] _host: Option, +) -> Result<(), Error> +where + P: NodePermissions + 'static, +{ + // TODO: hook up to InspectorServer + /* + let server = state.borrow_mut::(); + if let Some(host) = host { + server.set_host(host); + } + if let Some(port) = port { + server.set_port(port); + } + state + .borrow_mut::

() + .check_net((server.host(), Some(server.port())), "inspector.open")?; + */ + + Ok(()) +} + +#[op2(fast)] +pub fn op_inspector_close() { + // TODO: hook up to InspectorServer +} + +#[op2] +#[string] +pub fn op_inspector_url() -> Option { + // TODO: hook up to InspectorServer + None +} + +#[op2(fast)] +pub fn op_inspector_wait(state: &OpState) -> bool { + match state.try_borrow::>>() { + Some(inspector) => { + inspector + .borrow_mut() + .wait_for_session_and_break_on_next_statement(); + true + } + None => false, + } +} + +#[op2(fast)] +pub fn op_inspector_emit_protocol_event( + #[string] _event_name: String, + #[string] _params: String, +) { + // TODO: inspector channel & protocol notifications +} + +struct JSInspectorSession { + tx: RefCell>>, +} + +impl GarbageCollected for JSInspectorSession {} + +#[op2(stack_trace)] +#[cppgc] +pub fn op_inspector_connect<'s, P>( + isolate: *mut v8::Isolate, + scope: &mut v8::HandleScope<'s>, + state: &mut OpState, + connect_to_main_thread: bool, + callback: v8::Local<'s, v8::Function>, +) -> Result +where + P: NodePermissions + 'static, +{ + state + .borrow_mut::

() + .check_sys("inspector", "inspector.Session.connect")?; + + if connect_to_main_thread { + return Err(generic_error("connectToMainThread not supported")); + } + + let context = scope.get_current_context(); + let context = v8::Global::new(scope, context); + let callback = v8::Global::new(scope, callback); + + let inspector = state + .borrow::>>() + .borrow_mut(); + + let tx = inspector.create_raw_session( + InspectorSessionOptions { + kind: InspectorSessionKind::NonBlocking { + wait_for_disconnect: false, + }, + }, + // The inspector connection does not keep the event loop alive but + // when the inspector sends a message to the frontend, the JS that + // that runs may keep the event loop alive so we have to call back + // synchronously, instead of using the usual LocalInspectorSession + // UnboundedReceiver API. + Box::new(move |message| { + // SAFETY: This function is called directly by the inspector, so + // 1) The isolate is still valid + // 2) We are on the same thread as the Isolate + let scope = unsafe { &mut v8::CallbackScope::new(&mut *isolate) }; + let context = v8::Local::new(scope, context.clone()); + let scope = &mut v8::ContextScope::new(scope, context); + let scope = &mut v8::TryCatch::new(scope); + let recv = v8::undefined(scope); + if let Some(message) = v8::String::new(scope, &message.content) { + let callback = v8::Local::new(scope, callback.clone()); + callback.call(scope, recv.into(), &[message.into()]); + } + }), + ); + + Ok(JSInspectorSession { + tx: RefCell::new(Some(tx)), + }) +} + +#[op2(fast)] +pub fn op_inspector_dispatch( + #[cppgc] session: &JSInspectorSession, + #[string] message: String, +) { + if let Some(tx) = &*session.tx.borrow() { + let _ = tx.unbounded_send(message); + } +} + +#[op2(fast)] +pub fn op_inspector_disconnect(#[cppgc] session: &JSInspectorSession) { + drop(session.tx.borrow_mut().take()); +} diff --git a/ext/node/ops/ipc.rs b/ext/node/ops/ipc.rs index 59b6fece146352..672cf0d70709ef 100644 --- a/ext/node/ops/ipc.rs +++ b/ext/node/ops/ipc.rs @@ -17,8 +17,6 @@ mod impl_ { use std::task::Context; use std::task::Poll; - use deno_core::error::bad_resource_id; - use deno_core::error::AnyError; use deno_core::op2; use deno_core::serde; use deno_core::serde::Serializer; @@ -167,7 +165,7 @@ mod impl_ { #[smi] pub fn op_node_child_ipc_pipe( state: &mut OpState, - ) -> Result, AnyError> { + ) -> Result, io::Error> { let fd = match state.try_borrow_mut::() { Some(child_pipe_fd) => child_pipe_fd.0, None => return Ok(None), @@ -180,6 +178,18 @@ mod impl_ { )) } + #[derive(Debug, thiserror::Error)] + pub enum IpcError { + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error(transparent)] + IpcJsonStream(#[from] IpcJsonStreamError), + #[error(transparent)] + Canceled(#[from] deno_core::Canceled), + #[error("failed to serialize json value: {0}")] + SerdeJson(serde_json::Error), + } + #[op2(async)] pub fn op_node_ipc_write<'a>( scope: &mut v8::HandleScope<'a>, @@ -192,34 +202,37 @@ mod impl_ { // ideally we would just return `Result<(impl Future, bool), ..>`, but that's not // supported by `op2` currently. queue_ok: v8::Local<'a, v8::Array>, - ) -> Result>, AnyError> { + ) -> Result>, IpcError> { let mut serialized = Vec::with_capacity(64); let mut ser = serde_json::Serializer::new(&mut serialized); - serialize_v8_value(scope, value, &mut ser).map_err(|e| { - deno_core::error::type_error(format!( - "failed to serialize json value: {e}" - )) - })?; + serialize_v8_value(scope, value, &mut ser).map_err(IpcError::SerdeJson)?; serialized.push(b'\n'); let stream = state .borrow() .resource_table .get::(rid) - .map_err(|_| bad_resource_id())?; + .map_err(IpcError::Resource)?; let old = stream .queued_bytes .fetch_add(serialized.len(), std::sync::atomic::Ordering::Relaxed); if old + serialized.len() > 2 * INITIAL_CAPACITY { // sending messages too fast - let v = false.to_v8(scope)?; + let v = false.to_v8(scope).unwrap(); // Infallible queue_ok.set_index(scope, 0, v); } Ok(async move { - stream.clone().write_msg_bytes(&serialized).await?; + let cancel = stream.cancel.clone(); + let result = stream + .clone() + .write_msg_bytes(&serialized) + .or_cancel(cancel) + .await; + // adjust count even on error stream .queued_bytes .fetch_sub(serialized.len(), std::sync::atomic::Ordering::Relaxed); + result??; Ok(()) }) } @@ -239,12 +252,12 @@ mod impl_ { pub async fn op_node_ipc_read( state: Rc>, #[smi] rid: ResourceId, - ) -> Result { + ) -> Result { let stream = state .borrow() .resource_table .get::(rid) - .map_err(|_| bad_resource_id())?; + .map_err(IpcError::Resource)?; let cancel = stream.cancel.clone(); let mut stream = RcRef::map(stream, |r| &r.read_half).borrow_mut().await; @@ -400,7 +413,7 @@ mod impl_ { async fn write_msg_bytes( self: Rc, msg: &[u8], - ) -> Result<(), AnyError> { + ) -> Result<(), io::Error> { let mut write_half = RcRef::map(self, |r| &r.write_half).borrow_mut().await; write_half.write_all(msg).await?; @@ -455,6 +468,14 @@ mod impl_ { } } + #[derive(Debug, thiserror::Error)] + pub enum IpcJsonStreamError { + #[error("{0}")] + Io(#[source] std::io::Error), + #[error("{0}")] + SimdJson(#[source] simd_json::Error), + } + // JSON serialization stream over IPC pipe. // // `\n` is used as a delimiter between messages. @@ -475,7 +496,7 @@ mod impl_ { async fn read_msg( &mut self, - ) -> Result, AnyError> { + ) -> Result, IpcJsonStreamError> { let mut json = None; let nread = read_msg_inner( &mut self.pipe, @@ -483,7 +504,8 @@ mod impl_ { &mut json, &mut self.read_buffer, ) - .await?; + .await + .map_err(IpcJsonStreamError::Io)?; if nread == 0 { // EOF. return Ok(None); @@ -493,7 +515,8 @@ mod impl_ { Some(v) => v, None => { // Took more than a single read and some buffering. - simd_json::from_slice(&mut self.buffer[..nread])? + simd_json::from_slice(&mut self.buffer[..nread]) + .map_err(IpcJsonStreamError::SimdJson)? } }; diff --git a/ext/node/ops/mod.rs b/ext/node/ops/mod.rs index b562261f3991c9..e5ea8b41722161 100644 --- a/ext/node/ops/mod.rs +++ b/ext/node/ops/mod.rs @@ -7,8 +7,10 @@ pub mod fs; pub mod http; pub mod http2; pub mod idna; +pub mod inspector; pub mod ipc; pub mod os; +pub mod perf_hooks; pub mod process; pub mod require; pub mod tls; diff --git a/ext/node/ops/os/mod.rs b/ext/node/ops/os/mod.rs index ca91895f2244e2..ddb2a70c64883b 100644 --- a/ext/node/ops/os/mod.rs +++ b/ext/node/ops/os/mod.rs @@ -1,19 +1,31 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::mem::MaybeUninit; + use crate::NodePermissions; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; mod cpus; -mod priority; +pub mod priority; + +#[derive(Debug, thiserror::Error)] +pub enum OsError { + #[error(transparent)] + Priority(priority::PriorityError), + #[error(transparent)] + Permission(#[from] deno_permissions::PermissionCheckError), + #[error("Failed to get cpu info")] + FailedToGetCpuInfo, + #[error("Failed to get user info")] + FailedToGetUserInfo(#[source] std::io::Error), +} -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_node_os_get_priority

( state: &mut OpState, pid: u32, -) -> Result +) -> Result where P: NodePermissions + 'static, { @@ -22,15 +34,15 @@ where permissions.check_sys("getPriority", "node:os.getPriority()")?; } - priority::get_priority(pid) + priority::get_priority(pid).map_err(OsError::Priority) } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_node_os_set_priority

( state: &mut OpState, pid: u32, priority: i32, -) -> Result<(), AnyError> +) -> Result<(), OsError> where P: NodePermissions + 'static, { @@ -39,25 +51,171 @@ where permissions.check_sys("setPriority", "node:os.setPriority()")?; } - priority::set_priority(pid, priority) + priority::set_priority(pid, priority).map_err(OsError::Priority) } -#[op2] -#[string] -pub fn op_node_os_username

(state: &mut OpState) -> Result +#[derive(serde::Serialize)] +pub struct UserInfo { + username: String, + homedir: String, + shell: Option, +} + +#[cfg(unix)] +fn get_user_info(uid: u32) -> Result { + use std::ffi::CStr; + let mut pw: MaybeUninit = MaybeUninit::uninit(); + let mut result: *mut libc::passwd = std::ptr::null_mut(); + // SAFETY: libc call, no invariants + let max_buf_size = unsafe { libc::sysconf(libc::_SC_GETPW_R_SIZE_MAX) }; + let buf_size = if max_buf_size < 0 { + // from the man page + 16_384 + } else { + max_buf_size as usize + }; + let mut buf = { + let mut b = Vec::>::with_capacity(buf_size); + // SAFETY: MaybeUninit has no initialization invariants, and len == cap + unsafe { + b.set_len(buf_size); + } + b + }; + // SAFETY: libc call, args are correct + let s = unsafe { + libc::getpwuid_r( + uid, + pw.as_mut_ptr(), + buf.as_mut_ptr().cast(), + buf_size, + std::ptr::addr_of_mut!(result), + ) + }; + if result.is_null() { + if s != 0 { + return Err( + OsError::FailedToGetUserInfo(std::io::Error::last_os_error()), + ); + } else { + return Err(OsError::FailedToGetUserInfo(std::io::Error::from( + std::io::ErrorKind::NotFound, + ))); + } + } + // SAFETY: pw was initialized by the call to `getpwuid_r` above + let pw = unsafe { pw.assume_init() }; + // SAFETY: initialized above, pw alive until end of function, nul terminated + let username = unsafe { CStr::from_ptr(pw.pw_name) }; + // SAFETY: initialized above, pw alive until end of function, nul terminated + let homedir = unsafe { CStr::from_ptr(pw.pw_dir) }; + // SAFETY: initialized above, pw alive until end of function, nul terminated + let shell = unsafe { CStr::from_ptr(pw.pw_shell) }; + Ok(UserInfo { + username: username.to_string_lossy().into_owned(), + homedir: homedir.to_string_lossy().into_owned(), + shell: Some(shell.to_string_lossy().into_owned()), + }) +} + +#[cfg(windows)] +fn get_user_info(_uid: u32) -> Result { + use std::ffi::OsString; + use std::os::windows::ffi::OsStringExt; + + use windows_sys::Win32::Foundation::CloseHandle; + use windows_sys::Win32::Foundation::GetLastError; + use windows_sys::Win32::Foundation::ERROR_INSUFFICIENT_BUFFER; + use windows_sys::Win32::Foundation::HANDLE; + use windows_sys::Win32::System::Threading::GetCurrentProcess; + use windows_sys::Win32::System::Threading::OpenProcessToken; + use windows_sys::Win32::UI::Shell::GetUserProfileDirectoryW; + struct Handle(HANDLE); + impl Drop for Handle { + fn drop(&mut self) { + // SAFETY: win32 call + unsafe { + CloseHandle(self.0); + } + } + } + let mut token: MaybeUninit = MaybeUninit::uninit(); + + // Get a handle to the current process + // SAFETY: win32 call + unsafe { + if OpenProcessToken( + GetCurrentProcess(), + windows_sys::Win32::Security::TOKEN_READ, + token.as_mut_ptr(), + ) == 0 + { + return Err( + OsError::FailedToGetUserInfo(std::io::Error::last_os_error()), + ); + } + } + + // SAFETY: initialized by call above + let token = Handle(unsafe { token.assume_init() }); + + let mut bufsize = 0; + // get the size for the homedir buf (it'll end up in `bufsize`) + // SAFETY: win32 call + unsafe { + GetUserProfileDirectoryW(token.0, std::ptr::null_mut(), &mut bufsize); + let err = GetLastError(); + if err != ERROR_INSUFFICIENT_BUFFER { + return Err(OsError::FailedToGetUserInfo( + std::io::Error::from_raw_os_error(err as i32), + )); + } + } + let mut path = vec![0; bufsize as usize]; + // Actually get the homedir + // SAFETY: path is `bufsize` elements + unsafe { + if GetUserProfileDirectoryW(token.0, path.as_mut_ptr(), &mut bufsize) == 0 { + return Err( + OsError::FailedToGetUserInfo(std::io::Error::last_os_error()), + ); + } + } + // remove trailing nul + path.pop(); + let homedir_wide = OsString::from_wide(&path); + let homedir = homedir_wide.to_string_lossy().into_owned(); + + Ok(UserInfo { + username: deno_whoami::username(), + homedir, + shell: None, + }) +} + +#[op2(stack_trace)] +#[serde] +pub fn op_node_os_user_info

( + state: &mut OpState, + #[smi] uid: u32, +) -> Result where P: NodePermissions + 'static, { { let permissions = state.borrow_mut::

(); - permissions.check_sys("username", "node:os.userInfo()")?; + permissions + .check_sys("userInfo", "node:os.userInfo()") + .map_err(OsError::Permission)?; } - Ok(deno_whoami::username()) + get_user_info(uid) } -#[op2(fast)] -pub fn op_geteuid

(state: &mut OpState) -> Result +#[op2(fast, stack_trace)] +pub fn op_geteuid

( + state: &mut OpState, +) -> Result where P: NodePermissions + 'static, { @@ -75,8 +233,10 @@ where Ok(euid) } -#[op2(fast)] -pub fn op_getegid

(state: &mut OpState) -> Result +#[op2(fast, stack_trace)] +pub fn op_getegid

( + state: &mut OpState, +) -> Result where P: NodePermissions + 'static, { @@ -94,9 +254,9 @@ where Ok(egid) } -#[op2] +#[op2(stack_trace)] #[serde] -pub fn op_cpus

(state: &mut OpState) -> Result, AnyError> +pub fn op_cpus

(state: &mut OpState) -> Result, OsError> where P: NodePermissions + 'static, { @@ -105,12 +265,14 @@ where permissions.check_sys("cpus", "node:os.cpus()")?; } - cpus::cpu_info().ok_or_else(|| type_error("Failed to get cpu info")) + cpus::cpu_info().ok_or(OsError::FailedToGetCpuInfo) } -#[op2] +#[op2(stack_trace)] #[string] -pub fn op_homedir

(state: &mut OpState) -> Result, AnyError> +pub fn op_homedir

( + state: &mut OpState, +) -> Result, deno_core::error::AnyError> where P: NodePermissions + 'static, { diff --git a/ext/node/ops/os/priority.rs b/ext/node/ops/os/priority.rs index 043928e2a668a5..9a1ebcca705c6a 100644 --- a/ext/node/ops/os/priority.rs +++ b/ext/node/ops/os/priority.rs @@ -1,12 +1,18 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::AnyError; - pub use impl_::*; +#[derive(Debug, thiserror::Error)] +pub enum PriorityError { + #[error("{0}")] + Io(#[from] std::io::Error), + #[cfg(windows)] + #[error("Invalid priority")] + InvalidPriority, +} + #[cfg(unix)] mod impl_ { - use super::*; use errno::errno; use errno::set_errno; use errno::Errno; @@ -16,7 +22,7 @@ mod impl_ { const PRIORITY_HIGH: i32 = -14; // Ref: https://github.com/libuv/libuv/blob/55376b044b74db40772e8a6e24d67a8673998e02/src/unix/core.c#L1533-L1547 - pub fn get_priority(pid: u32) -> Result { + pub fn get_priority(pid: u32) -> Result { set_errno(Errno(0)); match ( // SAFETY: libc::getpriority is unsafe @@ -29,7 +35,10 @@ mod impl_ { } } - pub fn set_priority(pid: u32, priority: i32) -> Result<(), AnyError> { + pub fn set_priority( + pid: u32, + priority: i32, + ) -> Result<(), super::PriorityError> { // SAFETY: libc::setpriority is unsafe match unsafe { libc::setpriority(PRIO_PROCESS, pid as id_t, priority) } { -1 => Err(std::io::Error::last_os_error().into()), @@ -40,8 +49,6 @@ mod impl_ { #[cfg(windows)] mod impl_ { - use super::*; - use deno_core::error::type_error; use winapi::shared::minwindef::DWORD; use winapi::shared::minwindef::FALSE; use winapi::shared::ntdef::NULL; @@ -67,7 +74,7 @@ mod impl_ { const PRIORITY_HIGHEST: i32 = -20; // Ported from: https://github.com/libuv/libuv/blob/a877ca2435134ef86315326ef4ef0c16bdbabf17/src/win/util.c#L1649-L1685 - pub fn get_priority(pid: u32) -> Result { + pub fn get_priority(pid: u32) -> Result { // SAFETY: Windows API calls unsafe { let handle = if pid == 0 { @@ -95,7 +102,10 @@ mod impl_ { } // Ported from: https://github.com/libuv/libuv/blob/a877ca2435134ef86315326ef4ef0c16bdbabf17/src/win/util.c#L1688-L1719 - pub fn set_priority(pid: u32, priority: i32) -> Result<(), AnyError> { + pub fn set_priority( + pid: u32, + priority: i32, + ) -> Result<(), super::PriorityError> { // SAFETY: Windows API calls unsafe { let handle = if pid == 0 { @@ -109,7 +119,7 @@ mod impl_ { #[allow(clippy::manual_range_contains)] let priority_class = if priority < PRIORITY_HIGHEST || priority > PRIORITY_LOW { - return Err(type_error("Invalid priority")); + return Err(super::PriorityError::InvalidPriority); } else if priority < PRIORITY_HIGH { REALTIME_PRIORITY_CLASS } else if priority < PRIORITY_ABOVE_NORMAL { diff --git a/ext/node/ops/perf_hooks.rs b/ext/node/ops/perf_hooks.rs new file mode 100644 index 00000000000000..636d0b2adbb6b0 --- /dev/null +++ b/ext/node/ops/perf_hooks.rs @@ -0,0 +1,135 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use deno_core::op2; +use deno_core::GarbageCollected; + +use std::cell::Cell; + +#[derive(Debug, thiserror::Error)] +pub enum PerfHooksError { + #[error(transparent)] + TokioEld(#[from] tokio_eld::Error), +} + +pub struct EldHistogram { + eld: tokio_eld::EldHistogram, + started: Cell, +} + +impl GarbageCollected for EldHistogram {} + +#[op2] +impl EldHistogram { + // Creates an interval EldHistogram object that samples and reports the event + // loop delay over time. + // + // The delays will be reported in nanoseconds. + #[constructor] + #[cppgc] + pub fn new(#[smi] resolution: u32) -> Result { + Ok(EldHistogram { + eld: tokio_eld::EldHistogram::new(resolution as usize)?, + started: Cell::new(false), + }) + } + + // Disables the update interval timer. + // + // Returns true if the timer was stopped, false if it was already stopped. + #[fast] + fn enable(&self) -> bool { + if self.started.get() { + return false; + } + + self.eld.start(); + self.started.set(true); + + true + } + + // Enables the update interval timer. + // + // Returns true if the timer was started, false if it was already started. + #[fast] + fn disable(&self) -> bool { + if !self.started.get() { + return false; + } + + self.eld.stop(); + self.started.set(false); + + true + } + + // Returns the value at the given percentile. + // + // `percentile` ∈ (0, 100] + #[fast] + #[number] + fn percentile(&self, percentile: f64) -> u64 { + self.eld.value_at_percentile(percentile) + } + + // Returns the value at the given percentile as a bigint. + #[fast] + #[bigint] + fn percentile_big_int(&self, percentile: f64) -> u64 { + self.eld.value_at_percentile(percentile) + } + + // The number of samples recorded by the histogram. + #[getter] + #[number] + fn count(&self) -> u64 { + self.eld.len() + } + + // The number of samples recorded by the histogram as a bigint. + #[getter] + #[bigint] + fn count_big_int(&self) -> u64 { + self.eld.len() + } + + // The maximum recorded event loop delay. + #[getter] + #[number] + fn max(&self) -> u64 { + self.eld.max() + } + + // The maximum recorded event loop delay as a bigint. + #[getter] + #[bigint] + fn max_big_int(&self) -> u64 { + self.eld.max() + } + + // The mean of the recorded event loop delays. + #[getter] + fn mean(&self) -> f64 { + self.eld.mean() + } + + // The minimum recorded event loop delay. + #[getter] + #[number] + fn min(&self) -> u64 { + self.eld.min() + } + + // The minimum recorded event loop delay as a bigint. + #[getter] + #[bigint] + fn min_big_int(&self) -> u64 { + self.eld.min() + } + + // The standard deviation of the recorded event loop delays. + #[getter] + fn stddev(&self) -> f64 { + self.eld.stdev() + } +} diff --git a/ext/node/ops/process.rs b/ext/node/ops/process.rs index 0992c46c623d74..45c599bee24fbf 100644 --- a/ext/node/ops/process.rs +++ b/ext/node/ops/process.rs @@ -1,6 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; use deno_permissions::PermissionsContainer; @@ -46,12 +45,12 @@ fn kill(pid: i32, _sig: i32) -> i32 { } } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_node_process_kill( state: &mut OpState, #[smi] pid: i32, #[smi] sig: i32, -) -> Result { +) -> Result { state .borrow_mut::() .check_run_all("process.kill")?; diff --git a/ext/node/ops/require.rs b/ext/node/ops/require.rs index 3578719d0c339b..1c204f54e81bd6 100644 --- a/ext/node/ops/require.rs +++ b/ext/node/ops/require.rs @@ -1,40 +1,78 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::anyhow::Context; -use deno_core::error::generic_error; +use boxed_error::Boxed; use deno_core::error::AnyError; -use deno_core::normalize_path; use deno_core::op2; use deno_core::url::Url; use deno_core::v8; use deno_core::JsRuntimeInspector; -use deno_core::ModuleSpecifier; use deno_core::OpState; use deno_fs::FileSystemRc; +use deno_fs::V8MaybeStaticStr; use deno_package_json::PackageJsonRc; -use node_resolver::NodeModuleKind; -use node_resolver::NodeResolutionMode; +use deno_path_util::normalize_path; +use deno_path_util::url_from_file_path; +use deno_path_util::url_to_file_path; +use node_resolver::errors::ClosestPkgJsonError; +use node_resolver::NodeResolutionKind; +use node_resolver::ResolutionMode; use node_resolver::REQUIRE_CONDITIONS; +use std::borrow::Cow; use std::cell::RefCell; use std::path::Path; use std::path::PathBuf; use std::rc::Rc; use crate::NodePermissions; -use crate::NodeRequireResolverRc; +use crate::NodeRequireLoaderRc; use crate::NodeResolverRc; -use crate::NpmResolverRc; +use crate::NpmPackageFolderResolverRc; +use crate::PackageJsonResolverRc; -fn ensure_read_permission

( +#[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] +fn ensure_read_permission<'a, P>( state: &mut OpState, - file_path: &Path, -) -> Result<(), AnyError> + file_path: &'a Path, +) -> Result, deno_core::error::AnyError> where P: NodePermissions + 'static, { - let resolver = state.borrow::().clone(); + let loader = state.borrow::().clone(); let permissions = state.borrow_mut::

(); - resolver.ensure_read_permission(permissions, file_path) + loader.ensure_read_permission(permissions, file_path) +} + +#[derive(Debug, Boxed)] +pub struct RequireError(pub Box); + +#[derive(Debug, thiserror::Error)] +pub enum RequireErrorKind { + #[error(transparent)] + UrlParse(#[from] url::ParseError), + #[error(transparent)] + Permission(deno_core::error::AnyError), + #[error(transparent)] + PackageExportsResolve( + #[from] node_resolver::errors::PackageExportsResolveError, + ), + #[error(transparent)] + PackageJsonLoad(#[from] node_resolver::errors::PackageJsonLoadError), + #[error(transparent)] + ClosestPkgJson(#[from] node_resolver::errors::ClosestPkgJsonError), + #[error(transparent)] + PackageImportsResolve( + #[from] node_resolver::errors::PackageImportsResolveError, + ), + #[error(transparent)] + FilePathConversion(#[from] deno_path_util::UrlToFilePathError), + #[error(transparent)] + UrlConversion(#[from] deno_path_util::PathToUrlError), + #[error(transparent)] + Fs(#[from] deno_io::fs::FsError), + #[error(transparent)] + ReadModule(deno_core::error::AnyError), + #[error("Unable to get CWD: {0}")] + UnableToGetCwd(deno_io::fs::FsError), } #[op2] @@ -88,12 +126,12 @@ pub fn op_require_init_paths() -> Vec { vec![] } -#[op2] +#[op2(stack_trace)] #[serde] pub fn op_require_node_module_paths

( state: &mut OpState, #[string] from: String, -) -> Result, AnyError> +) -> Result, RequireError> where P: NodePermissions + 'static, { @@ -102,13 +140,10 @@ where let from = if from.starts_with("file:///") { url_to_file_path(&Url::parse(&from)?)? } else { - let current_dir = - &(fs.cwd().map_err(AnyError::from)).context("Unable to get CWD")?; - deno_core::normalize_path(current_dir.join(from)) + let current_dir = &fs.cwd().map_err(RequireErrorKind::UnableToGetCwd)?; + normalize_path(current_dir.join(from)) }; - ensure_read_permission::

(state, &from)?; - if cfg!(windows) { // return root node_modules when path is 'D:\\'. let from_str = from.to_str().unwrap(); @@ -189,17 +224,17 @@ pub fn op_require_resolve_deno_dir( state: &mut OpState, #[string] request: String, #[string] parent_filename: String, -) -> Option { - let resolver = state.borrow::(); - resolver - .resolve_package_folder_from_package( - &request, - &ModuleSpecifier::from_file_path(&parent_filename).unwrap_or_else(|_| { - panic!("Url::from_file_path: [{:?}]", parent_filename) - }), - ) - .ok() - .map(|p| p.to_string_lossy().into_owned()) +) -> Result, AnyError> { + let resolver = state.borrow::(); + Ok( + resolver + .resolve_package_folder_from_package( + &request, + &url_from_file_path(&PathBuf::from(parent_filename))?, + ) + .ok() + .map(|p| p.to_string_lossy().into_owned()), + ) } #[op2(fast)] @@ -207,8 +242,11 @@ pub fn op_require_is_deno_dir_package( state: &mut OpState, #[string] path: String, ) -> bool { - let resolver = state.borrow::(); - resolver.in_npm_package_at_file_path(&PathBuf::from(path)) + let resolver = state.borrow::(); + match deno_path_util::url_from_file_path(&PathBuf::from(path)) { + Ok(specifier) => resolver.in_npm_package(&specifier), + Err(_) => false, + } } #[op2] @@ -258,16 +296,16 @@ pub fn op_require_path_is_absolute(#[string] p: String) -> bool { PathBuf::from(p).is_absolute() } -#[op2(fast)] +#[op2(fast, stack_trace)] pub fn op_require_stat

( state: &mut OpState, #[string] path: String, -) -> Result +) -> Result where P: NodePermissions + 'static, { let path = PathBuf::from(path); - ensure_read_permission::

(state, &path)?; + let path = ensure_read_permission::

(state, &path)?; let fs = state.borrow::(); if let Ok(metadata) = fs.stat_sync(&path) { if metadata.is_file { @@ -280,20 +318,21 @@ where Ok(-1) } -#[op2] +#[op2(stack_trace)] #[string] pub fn op_require_real_path

( state: &mut OpState, #[string] request: String, -) -> Result +) -> Result where P: NodePermissions + 'static, { let path = PathBuf::from(request); - ensure_read_permission::

(state, &path)?; + let path = ensure_read_permission::

(state, &path) + .map_err(RequireErrorKind::Permission)?; let fs = state.borrow::(); let canonicalized_path = - deno_core::strip_unc_prefix(fs.realpath_sync(&path)?); + deno_path_util::strip_unc_prefix(fs.realpath_sync(&path)?); Ok(canonicalized_path.to_string_lossy().into_owned()) } @@ -317,12 +356,14 @@ pub fn op_require_path_resolve(#[serde] parts: Vec) -> String { #[string] pub fn op_require_path_dirname( #[string] request: String, -) -> Result { +) -> Result { let p = PathBuf::from(request); if let Some(parent) = p.parent() { Ok(parent.to_string_lossy().into_owned()) } else { - Err(generic_error("Path doesn't have a parent")) + Err(deno_core::error::generic_error( + "Path doesn't have a parent", + )) } } @@ -330,23 +371,25 @@ pub fn op_require_path_dirname( #[string] pub fn op_require_path_basename( #[string] request: String, -) -> Result { +) -> Result { let p = PathBuf::from(request); if let Some(path) = p.file_name() { Ok(path.to_string_lossy().into_owned()) } else { - Err(generic_error("Path doesn't have a file name")) + Err(deno_core::error::generic_error( + "Path doesn't have a file name", + )) } } -#[op2] +#[op2(stack_trace)] #[string] pub fn op_require_try_self_parent_path

( state: &mut OpState, has_parent: bool, #[string] maybe_parent_filename: Option, #[string] maybe_parent_id: Option, -) -> Result, AnyError> +) -> Result, deno_core::error::AnyError> where P: NodePermissions + 'static, { @@ -362,7 +405,7 @@ where if parent_id == "" || parent_id == "internal/preload" { let fs = state.borrow::(); if let Ok(cwd) = fs.cwd() { - ensure_read_permission::

(state, &cwd)?; + let cwd = ensure_read_permission::

(state, &cwd)?; return Ok(Some(cwd.to_string_lossy().into_owned())); } } @@ -370,13 +413,13 @@ where Ok(None) } -#[op2] +#[op2(stack_trace)] #[string] pub fn op_require_try_self

( state: &mut OpState, #[string] parent_path: Option, #[string] request: String, -) -> Result, AnyError> +) -> Result, RequireError> where P: NodePermissions + 'static, { @@ -384,8 +427,8 @@ where return Ok(None); } - let node_resolver = state.borrow::(); - let pkg = node_resolver + let pkg_json_resolver = state.borrow::(); + let pkg = pkg_json_resolver .get_closest_package_json_from_path(&PathBuf::from(parent_path.unwrap())) .ok() .flatten(); @@ -414,14 +457,15 @@ where let referrer = deno_core::url::Url::from_file_path(&pkg.path).unwrap(); if let Some(exports) = &pkg.exports { + let node_resolver = state.borrow::(); let r = node_resolver.package_exports_resolve( &pkg.path, &expansion, exports, Some(&referrer), - NodeModuleKind::Cjs, + ResolutionMode::Require, REQUIRE_CONDITIONS, - NodeResolutionMode::Execution, + NodeResolutionKind::Execution, )?; Ok(Some(if r.scheme() == "file" { url_to_file_path_string(&r)? @@ -433,19 +477,24 @@ where } } -#[op2] -#[string] +#[op2(stack_trace)] +#[to_v8] pub fn op_require_read_file

( state: &mut OpState, #[string] file_path: String, -) -> Result +) -> Result where P: NodePermissions + 'static, { let file_path = PathBuf::from(file_path); - ensure_read_permission::

(state, &file_path)?; - let fs = state.borrow::(); - Ok(fs.read_text_file_lossy_sync(&file_path, None)?) + // todo(dsherret): there's multiple borrows to NodeRequireLoaderRc here + let file_path = ensure_read_permission::

(state, &file_path) + .map_err(RequireErrorKind::Permission)?; + let loader = state.borrow::(); + loader + .load_text_file_lossy(&file_path) + .map(V8MaybeStaticStr) + .map_err(|e| RequireErrorKind::ReadModule(e).into_box()) } #[op2] @@ -460,7 +509,7 @@ pub fn op_require_as_file_path(#[string] file_or_url: String) -> String { file_or_url } -#[op2] +#[op2(stack_trace)] #[string] pub fn op_require_resolve_exports

( state: &mut OpState, @@ -470,16 +519,17 @@ pub fn op_require_resolve_exports

( #[string] name: String, #[string] expansion: String, #[string] parent_path: String, -) -> Result, AnyError> +) -> Result, RequireError> where P: NodePermissions + 'static, { let fs = state.borrow::(); - let npm_resolver = state.borrow::(); let node_resolver = state.borrow::(); + let pkg_json_resolver = state.borrow::(); let modules_path = PathBuf::from(&modules_path_str); - let pkg_path = if npm_resolver.in_npm_package_at_file_path(&modules_path) + let modules_specifier = deno_path_util::url_from_file_path(&modules_path)?; + let pkg_path = if node_resolver.in_npm_package(&modules_specifier) && !uses_local_node_modules_dir { modules_path @@ -493,7 +543,7 @@ where } }; let Some(pkg) = - node_resolver.load_package_json(&pkg_path.join("package.json"))? + pkg_json_resolver.load_package_json(&pkg_path.join("package.json"))? else { return Ok(None); }; @@ -501,15 +551,19 @@ where return Ok(None); }; - let referrer = Url::from_file_path(parent_path).unwrap(); + let referrer = if parent_path.is_empty() { + None + } else { + Some(Url::from_file_path(parent_path).unwrap()) + }; let r = node_resolver.package_exports_resolve( &pkg.path, &format!(".{expansion}"), exports, - Some(&referrer), - NodeModuleKind::Cjs, + referrer.as_ref(), + ResolutionMode::Require, REQUIRE_CONDITIONS, - NodeResolutionMode::Execution, + NodeResolutionKind::Execution, )?; Ok(Some(if r.scheme() == "file" { url_to_file_path_string(&r)? @@ -518,24 +572,20 @@ where })) } -#[op2] -#[serde] -pub fn op_require_read_closest_package_json

( +#[op2(fast)] +pub fn op_require_is_maybe_cjs( state: &mut OpState, #[string] filename: String, -) -> Result, AnyError> -where - P: NodePermissions + 'static, -{ +) -> Result { let filename = PathBuf::from(filename); - ensure_read_permission::

(state, filename.parent().unwrap())?; - let node_resolver = state.borrow::().clone(); - node_resolver - .get_closest_package_json_from_path(&filename) - .map_err(AnyError::from) + let Ok(url) = url_from_file_path(&filename) else { + return Ok(false); + }; + let loader = state.borrow::(); + loader.is_maybe_cjs(&url) } -#[op2] +#[op2(stack_trace)] #[serde] pub fn op_require_read_package_scope

( state: &mut OpState, @@ -544,47 +594,48 @@ pub fn op_require_read_package_scope

( where P: NodePermissions + 'static, { - let node_resolver = state.borrow::().clone(); + let pkg_json_resolver = state.borrow::(); let package_json_path = PathBuf::from(package_json_path); if package_json_path.file_name() != Some("package.json".as_ref()) { // permissions: do not allow reading a non-package.json file return None; } - node_resolver + pkg_json_resolver .load_package_json(&package_json_path) .ok() .flatten() } -#[op2] +#[op2(stack_trace)] #[string] pub fn op_require_package_imports_resolve

( state: &mut OpState, #[string] referrer_filename: String, #[string] request: String, -) -> Result, AnyError> +) -> Result, RequireError> where P: NodePermissions + 'static, { let referrer_path = PathBuf::from(&referrer_filename); - ensure_read_permission::

(state, &referrer_path)?; - let node_resolver = state.borrow::(); + let referrer_path = ensure_read_permission::

(state, &referrer_path) + .map_err(RequireErrorKind::Permission)?; + let pkg_json_resolver = state.borrow::(); let Some(pkg) = - node_resolver.get_closest_package_json_from_path(&referrer_path)? + pkg_json_resolver.get_closest_package_json_from_path(&referrer_path)? else { return Ok(None); }; if pkg.imports.is_some() { - let referrer_url = - deno_core::url::Url::from_file_path(&referrer_filename).unwrap(); + let node_resolver = state.borrow::(); + let referrer_url = Url::from_file_path(&referrer_filename).unwrap(); let url = node_resolver.package_imports_resolve( &request, Some(&referrer_url), - NodeModuleKind::Cjs, + ResolutionMode::Require, Some(&pkg), REQUIRE_CONDITIONS, - NodeResolutionMode::Execution, + NodeResolutionKind::Execution, )?; Ok(Some(url_to_file_path_string(&url)?)) } else { @@ -602,20 +653,11 @@ pub fn op_require_break_on_next_statement(state: Rc>) { inspector.wait_for_session_and_break_on_next_statement() } -fn url_to_file_path_string(url: &Url) -> Result { +fn url_to_file_path_string(url: &Url) -> Result { let file_path = url_to_file_path(url)?; Ok(file_path.to_string_lossy().into_owned()) } -fn url_to_file_path(url: &Url) -> Result { - match url.to_file_path() { - Ok(file_path) => Ok(file_path), - Err(()) => { - deno_core::anyhow::bail!("failed to convert '{}' to file path", url) - } - } -} - #[op2(fast)] pub fn op_require_can_parse_as_esm( scope: &mut v8::HandleScope, diff --git a/ext/node/ops/util.rs b/ext/node/ops/util.rs index 533d51c92c73b4..1c177ac0431d55 100644 --- a/ext/node/ops/util.rs +++ b/ext/node/ops/util.rs @@ -1,6 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; use deno_core::ResourceHandle; @@ -22,7 +21,7 @@ enum HandleType { pub fn op_node_guess_handle_type( state: &mut OpState, rid: u32, -) -> Result { +) -> Result { let handle = state.resource_table.get_handle(rid)?; let handle_type = match handle { diff --git a/ext/node/ops/v8.rs b/ext/node/ops/v8.rs index 8813d2e18efe6f..61f67f11f73a27 100644 --- a/ext/node/ops/v8.rs +++ b/ext/node/ops/v8.rs @@ -1,7 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::generic_error; -use deno_core::error::type_error; -use deno_core::error::AnyError; + use deno_core::op2; use deno_core::v8; use deno_core::FastString; @@ -206,10 +204,9 @@ pub fn op_v8_write_value( scope: &mut v8::HandleScope, #[cppgc] ser: &Serializer, value: v8::Local, -) -> Result<(), AnyError> { +) { let context = scope.get_current_context(); ser.inner.write_value(context, value); - Ok(()) } struct DeserBuffer { @@ -271,11 +268,13 @@ pub fn op_v8_new_deserializer( scope: &mut v8::HandleScope, obj: v8::Local, buffer: v8::Local, -) -> Result, AnyError> { +) -> Result, deno_core::error::AnyError> { let offset = buffer.byte_offset(); let len = buffer.byte_length(); let backing_store = buffer.get_backing_store().ok_or_else(|| { - generic_error("deserialization buffer has no backing store") + deno_core::error::generic_error( + "deserialization buffer has no backing store", + ) })?; let (buf_slice, buf_ptr) = if let Some(data) = backing_store.data() { // SAFETY: the offset is valid for the underlying buffer because we're getting it directly from v8 @@ -317,10 +316,10 @@ pub fn op_v8_transfer_array_buffer_de( #[op2(fast)] pub fn op_v8_read_double( #[cppgc] deser: &Deserializer, -) -> Result { +) -> Result { let mut double = 0f64; if !deser.inner.read_double(&mut double) { - return Err(type_error("ReadDouble() failed")); + return Err(deno_core::error::type_error("ReadDouble() failed")); } Ok(double) } @@ -355,10 +354,10 @@ pub fn op_v8_read_raw_bytes( #[op2(fast)] pub fn op_v8_read_uint32( #[cppgc] deser: &Deserializer, -) -> Result { +) -> Result { let mut value = 0; if !deser.inner.read_uint32(&mut value) { - return Err(type_error("ReadUint32() failed")); + return Err(deno_core::error::type_error("ReadUint32() failed")); } Ok(value) @@ -368,10 +367,10 @@ pub fn op_v8_read_uint32( #[serde] pub fn op_v8_read_uint64( #[cppgc] deser: &Deserializer, -) -> Result<(u32, u32), AnyError> { +) -> Result<(u32, u32), deno_core::error::AnyError> { let mut val = 0; if !deser.inner.read_uint64(&mut val) { - return Err(type_error("ReadUint64() failed")); + return Err(deno_core::error::type_error("ReadUint64() failed")); } Ok(((val >> 32) as u32, val as u32)) diff --git a/ext/node/ops/winerror.rs b/ext/node/ops/winerror.rs index c0d66f7d0b29f5..cb053774ef6288 100644 --- a/ext/node/ops/winerror.rs +++ b/ext/node/ops/winerror.rs @@ -62,10 +62,11 @@ pub fn op_node_sys_to_uv_error(err: i32) -> String { WSAEHOSTUNREACH => "EHOSTUNREACH", ERROR_INSUFFICIENT_BUFFER => "EINVAL", ERROR_INVALID_DATA => "EINVAL", - ERROR_INVALID_NAME => "EINVAL", + ERROR_INVALID_NAME => "ENOENT", ERROR_INVALID_PARAMETER => "EINVAL", WSAEINVAL => "EINVAL", WSAEPFNOSUPPORT => "EINVAL", + ERROR_NOT_A_REPARSE_POINT => "EINVAL", ERROR_BEGINNING_OF_MEDIA => "EIO", ERROR_BUS_RESET => "EIO", ERROR_CRC => "EIO", diff --git a/ext/node/ops/worker_threads.rs b/ext/node/ops/worker_threads.rs index c7ea4c52c26007..37a7b477d02317 100644 --- a/ext/node/ops/worker_threads.rs +++ b/ext/node/ops/worker_threads.rs @@ -1,37 +1,56 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::generic_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::url::Url; use deno_core::OpState; use deno_fs::FileSystemRc; -use node_resolver::NodeResolution; +use std::borrow::Cow; use std::path::Path; use std::path::PathBuf; use crate::NodePermissions; -use crate::NodeRequireResolverRc; -use crate::NodeResolverRc; +use crate::NodeRequireLoaderRc; -fn ensure_read_permission

( +#[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] +fn ensure_read_permission<'a, P>( state: &mut OpState, - file_path: &Path, -) -> Result<(), AnyError> + file_path: &'a Path, +) -> Result, deno_core::error::AnyError> where P: NodePermissions + 'static, { - let resolver = state.borrow::().clone(); + let loader = state.borrow::().clone(); let permissions = state.borrow_mut::

(); - resolver.ensure_read_permission(permissions, file_path) + loader.ensure_read_permission(permissions, file_path) } -#[op2] +#[derive(Debug, thiserror::Error)] +pub enum WorkerThreadsFilenameError { + #[error(transparent)] + Permission(deno_core::error::AnyError), + #[error("{0}")] + UrlParse(#[from] url::ParseError), + #[error("Relative path entries must start with '.' or '..'")] + InvalidRelativeUrl, + #[error("URL from Path-String")] + UrlFromPathString, + #[error("URL to Path-String")] + UrlToPathString, + #[error("URL to Path")] + UrlToPath, + #[error("File not found [{0:?}]")] + FileNotFound(PathBuf), + #[error(transparent)] + Fs(#[from] deno_io::fs::FsError), +} + +// todo(dsherret): we should remove this and do all this work inside op_create_worker +#[op2(stack_trace)] #[string] pub fn op_worker_threads_filename

( state: &mut OpState, #[string] specifier: String, -) -> Result +) -> Result where P: NodePermissions + 'static, { @@ -43,44 +62,26 @@ where } else { let path = PathBuf::from(&specifier); if path.is_relative() && !specifier.starts_with('.') { - return Err(generic_error( - "Relative path entries must start with '.' or '..'", - )); + return Err(WorkerThreadsFilenameError::InvalidRelativeUrl); } - ensure_read_permission::

(state, &path)?; + let path = ensure_read_permission::

(state, &path) + .map_err(WorkerThreadsFilenameError::Permission)?; let fs = state.borrow::(); let canonicalized_path = - deno_core::strip_unc_prefix(fs.realpath_sync(&path)?); + deno_path_util::strip_unc_prefix(fs.realpath_sync(&path)?); Url::from_file_path(canonicalized_path) - .map_err(|e| generic_error(format!("URL from Path-String: {:#?}", e)))? + .map_err(|_| WorkerThreadsFilenameError::UrlFromPathString)? }; let url_path = url .to_file_path() - .map_err(|e| generic_error(format!("URL to Path-String: {:#?}", e)))?; - ensure_read_permission::

(state, &url_path)?; + .map_err(|_| WorkerThreadsFilenameError::UrlToPathString)?; + let url_path = ensure_read_permission::

(state, &url_path) + .map_err(WorkerThreadsFilenameError::Permission)?; let fs = state.borrow::(); if !fs.exists_sync(&url_path) { - return Err(generic_error(format!("File not found [{:?}]", url_path))); - } - let node_resolver = state.borrow::(); - match node_resolver.url_to_node_resolution(url)? { - NodeResolution::Esm(u) => Ok(u.to_string()), - NodeResolution::CommonJs(u) => wrap_cjs(u), - NodeResolution::BuiltIn(_) => Err(generic_error("Neither ESM nor CJS")), + return Err(WorkerThreadsFilenameError::FileNotFound( + url_path.to_path_buf(), + )); } -} - -/// -/// Wrap a CJS file-URL and the required setup in a stringified `data:`-URL -/// -fn wrap_cjs(url: Url) -> Result { - let path = url - .to_file_path() - .map_err(|e| generic_error(format!("URL to Path: {:#?}", e)))?; - let filename = path.file_name().unwrap().to_string_lossy(); - Ok(format!( - "data:text/javascript,import {{ createRequire }} from \"node:module\";\ - const require = createRequire(\"{}\"); require(\"./{}\");", - url, filename, - )) + Ok(url.to_string()) } diff --git a/ext/node/ops/zlib/brotli.rs b/ext/node/ops/zlib/brotli.rs index 3e3905fc3d6e10..1a681ff7f719a6 100644 --- a/ext/node/ops/zlib/brotli.rs +++ b/ext/node/ops/zlib/brotli.rs @@ -9,8 +9,6 @@ use brotli::BrotliDecompressStream; use brotli::BrotliResult; use brotli::BrotliState; use brotli::Decompressor; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::JsBuffer; use deno_core::OpState; @@ -19,7 +17,23 @@ use deno_core::ToJsBuffer; use std::cell::RefCell; use std::io::Read; -fn encoder_mode(mode: u32) -> Result { +#[derive(Debug, thiserror::Error)] +pub enum BrotliError { + #[error("Invalid encoder mode")] + InvalidEncoderMode, + #[error("Failed to compress")] + CompressFailed, + #[error("Failed to decompress")] + DecompressFailed, + #[error(transparent)] + Join(#[from] tokio::task::JoinError), + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error("{0}")] + Io(std::io::Error), +} + +fn encoder_mode(mode: u32) -> Result { Ok(match mode { 0 => BrotliEncoderMode::BROTLI_MODE_GENERIC, 1 => BrotliEncoderMode::BROTLI_MODE_TEXT, @@ -28,7 +42,7 @@ fn encoder_mode(mode: u32) -> Result { 4 => BrotliEncoderMode::BROTLI_FORCE_MSB_PRIOR, 5 => BrotliEncoderMode::BROTLI_FORCE_UTF8_PRIOR, 6 => BrotliEncoderMode::BROTLI_FORCE_SIGNED_PRIOR, - _ => return Err(type_error("Invalid encoder mode")), + _ => return Err(BrotliError::InvalidEncoderMode), }) } @@ -40,7 +54,7 @@ pub fn op_brotli_compress( #[smi] quality: i32, #[smi] lgwin: i32, #[smi] mode: u32, -) -> Result { +) -> Result { let mode = encoder_mode(mode)?; let mut out_size = out.len(); @@ -57,7 +71,7 @@ pub fn op_brotli_compress( &mut |_, _, _, _| (), ); if result != 1 { - return Err(type_error("Failed to compress")); + return Err(BrotliError::CompressFailed); } Ok(out_size) @@ -87,7 +101,7 @@ pub async fn op_brotli_compress_async( #[smi] quality: i32, #[smi] lgwin: i32, #[smi] mode: u32, -) -> Result { +) -> Result { let mode = encoder_mode(mode)?; tokio::task::spawn_blocking(move || { let input = &*input; @@ -107,7 +121,7 @@ pub async fn op_brotli_compress_async( &mut |_, _, _, _| (), ); if result != 1 { - return Err(type_error("Failed to compress")); + return Err(BrotliError::CompressFailed); } out.truncate(out_size); @@ -151,8 +165,11 @@ pub fn op_brotli_compress_stream( #[smi] rid: u32, #[buffer] input: &[u8], #[buffer] output: &mut [u8], -) -> Result { - let ctx = state.resource_table.get::(rid)?; +) -> Result { + let ctx = state + .resource_table + .get::(rid) + .map_err(BrotliError::Resource)?; let mut inst = ctx.inst.borrow_mut(); let mut output_offset = 0; @@ -168,7 +185,7 @@ pub fn op_brotli_compress_stream( &mut |_, _, _, _| (), ); if !result { - return Err(type_error("Failed to compress")); + return Err(BrotliError::CompressFailed); } Ok(output_offset) @@ -180,8 +197,11 @@ pub fn op_brotli_compress_stream_end( state: &mut OpState, #[smi] rid: u32, #[buffer] output: &mut [u8], -) -> Result { - let ctx = state.resource_table.get::(rid)?; +) -> Result { + let ctx = state + .resource_table + .get::(rid) + .map_err(BrotliError::Resource)?; let mut inst = ctx.inst.borrow_mut(); let mut output_offset = 0; @@ -197,13 +217,13 @@ pub fn op_brotli_compress_stream_end( &mut |_, _, _, _| (), ); if !result { - return Err(type_error("Failed to compress")); + return Err(BrotliError::CompressFailed); } Ok(output_offset) } -fn brotli_decompress(buffer: &[u8]) -> Result { +fn brotli_decompress(buffer: &[u8]) -> Result { let mut output = Vec::with_capacity(4096); let mut decompressor = Decompressor::new(buffer, buffer.len()); decompressor.read_to_end(&mut output)?; @@ -214,7 +234,7 @@ fn brotli_decompress(buffer: &[u8]) -> Result { #[serde] pub fn op_brotli_decompress( #[buffer] buffer: &[u8], -) -> Result { +) -> Result { brotli_decompress(buffer) } @@ -222,8 +242,11 @@ pub fn op_brotli_decompress( #[serde] pub async fn op_brotli_decompress_async( #[buffer] buffer: JsBuffer, -) -> Result { - tokio::task::spawn_blocking(move || brotli_decompress(&buffer)).await? +) -> Result { + tokio::task::spawn_blocking(move || { + brotli_decompress(&buffer).map_err(BrotliError::Io) + }) + .await? } struct BrotliDecompressCtx { @@ -252,8 +275,11 @@ pub fn op_brotli_decompress_stream( #[smi] rid: u32, #[buffer] input: &[u8], #[buffer] output: &mut [u8], -) -> Result { - let ctx = state.resource_table.get::(rid)?; +) -> Result { + let ctx = state + .resource_table + .get::(rid) + .map_err(BrotliError::Resource)?; let mut inst = ctx.inst.borrow_mut(); let mut output_offset = 0; @@ -268,7 +294,7 @@ pub fn op_brotli_decompress_stream( &mut inst, ); if matches!(result, BrotliResult::ResultFailure) { - return Err(type_error("Failed to decompress")); + return Err(BrotliError::DecompressFailed); } Ok(output_offset) @@ -280,8 +306,11 @@ pub fn op_brotli_decompress_stream_end( state: &mut OpState, #[smi] rid: u32, #[buffer] output: &mut [u8], -) -> Result { - let ctx = state.resource_table.get::(rid)?; +) -> Result { + let ctx = state + .resource_table + .get::(rid) + .map_err(BrotliError::Resource)?; let mut inst = ctx.inst.borrow_mut(); let mut output_offset = 0; @@ -296,7 +325,7 @@ pub fn op_brotli_decompress_stream_end( &mut inst, ); if matches!(result, BrotliResult::ResultFailure) { - return Err(type_error("Failed to decompress")); + return Err(BrotliError::DecompressFailed); } Ok(output_offset) diff --git a/ext/node/ops/zlib/mod.rs b/ext/node/ops/zlib/mod.rs index b1d6d21d22d60c..991c0925d2538f 100644 --- a/ext/node/ops/zlib/mod.rs +++ b/ext/node/ops/zlib/mod.rs @@ -1,14 +1,14 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::type_error; -use deno_core::error::AnyError; + use deno_core::op2; +use libc::c_ulong; use std::borrow::Cow; use std::cell::RefCell; use zlib::*; mod alloc; pub mod brotli; -mod mode; +pub mod mode; mod stream; use mode::Flush; @@ -17,11 +17,11 @@ use mode::Mode; use self::stream::StreamWrapper; #[inline] -fn check(condition: bool, msg: &str) -> Result<(), AnyError> { +fn check(condition: bool, msg: &str) -> Result<(), deno_core::error::AnyError> { if condition { Ok(()) } else { - Err(type_error(msg.to_string())) + Err(deno_core::error::type_error(msg.to_string())) } } @@ -56,7 +56,7 @@ impl ZlibInner { out_off: u32, out_len: u32, flush: Flush, - ) -> Result<(), AnyError> { + ) -> Result<(), deno_core::error::AnyError> { check(self.init_done, "write before init")?; check(!self.write_in_progress, "write already in progress")?; check(!self.pending_close, "close already in progress")?; @@ -65,11 +65,11 @@ impl ZlibInner { let next_in = input .get(in_off as usize..in_off as usize + in_len as usize) - .ok_or_else(|| type_error("invalid input range"))? + .ok_or_else(|| deno_core::error::type_error("invalid input range"))? .as_ptr() as *mut _; let next_out = out .get_mut(out_off as usize..out_off as usize + out_len as usize) - .ok_or_else(|| type_error("invalid output range"))? + .ok_or_else(|| deno_core::error::type_error("invalid output range"))? .as_mut_ptr(); self.strm.avail_in = in_len; @@ -81,7 +81,10 @@ impl ZlibInner { Ok(()) } - fn do_write(&mut self, flush: Flush) -> Result<(), AnyError> { + fn do_write( + &mut self, + flush: Flush, + ) -> Result<(), deno_core::error::AnyError> { self.flush = flush; match self.mode { Mode::Deflate | Mode::Gzip | Mode::DeflateRaw => { @@ -127,7 +130,7 @@ impl ZlibInner { self.mode = Mode::Inflate; } } else if next_expected_header_byte.is_some() { - return Err(type_error( + return Err(deno_core::error::type_error( "invalid number of gzip magic number bytes read", )); } @@ -181,7 +184,7 @@ impl ZlibInner { Ok(()) } - fn init_stream(&mut self) -> Result<(), AnyError> { + fn init_stream(&mut self) -> Result<(), deno_core::error::AnyError> { match self.mode { Mode::Gzip | Mode::Gunzip => self.window_bits += 16, Mode::Unzip => self.window_bits += 32, @@ -199,7 +202,7 @@ impl ZlibInner { Mode::Inflate | Mode::Gunzip | Mode::InflateRaw | Mode::Unzip => { self.strm.inflate_init(self.window_bits) } - Mode::None => return Err(type_error("Unknown mode")), + Mode::None => return Err(deno_core::error::type_error("Unknown mode")), }; self.write_in_progress = false; @@ -208,7 +211,7 @@ impl ZlibInner { Ok(()) } - fn close(&mut self) -> Result { + fn close(&mut self) -> Result { if self.write_in_progress { self.pending_close = true; return Ok(false); @@ -222,10 +225,8 @@ impl ZlibInner { Ok(true) } - fn reset_stream(&mut self) -> Result<(), AnyError> { + fn reset_stream(&mut self) { self.err = self.strm.reset(self.mode); - - Ok(()) } } @@ -243,7 +244,7 @@ impl deno_core::Resource for Zlib { #[op2] #[cppgc] -pub fn op_zlib_new(#[smi] mode: i32) -> Result { +pub fn op_zlib_new(#[smi] mode: i32) -> Result { let mode = Mode::try_from(mode)?; let inner = ZlibInner { @@ -256,12 +257,20 @@ pub fn op_zlib_new(#[smi] mode: i32) -> Result { }) } +#[derive(Debug, thiserror::Error)] +pub enum ZlibError { + #[error("zlib not initialized")] + NotInitialized, + #[error(transparent)] + Mode(#[from] mode::ModeError), + #[error(transparent)] + Other(#[from] deno_core::error::AnyError), +} + #[op2(fast)] -pub fn op_zlib_close(#[cppgc] resource: &Zlib) -> Result<(), AnyError> { +pub fn op_zlib_close(#[cppgc] resource: &Zlib) -> Result<(), ZlibError> { let mut resource = resource.inner.borrow_mut(); - let zlib = resource - .as_mut() - .ok_or_else(|| type_error("zlib not initialized"))?; + let zlib = resource.as_mut().ok_or(ZlibError::NotInitialized)?; // If there is a pending write, defer the close until the write is done. zlib.close()?; @@ -282,11 +291,9 @@ pub fn op_zlib_write( #[smi] out_off: u32, #[smi] out_len: u32, #[buffer] result: &mut [u32], -) -> Result { +) -> Result { let mut zlib = resource.inner.borrow_mut(); - let zlib = zlib - .as_mut() - .ok_or_else(|| type_error("zlib not initialized"))?; + let zlib = zlib.as_mut().ok_or(ZlibError::NotInitialized)?; let flush = Flush::try_from(flush)?; zlib.start_write(input, in_off, in_len, out, out_off, out_len, flush)?; @@ -307,11 +314,9 @@ pub fn op_zlib_init( #[smi] mem_level: i32, #[smi] strategy: i32, #[buffer] dictionary: &[u8], -) -> Result { +) -> Result { let mut zlib = resource.inner.borrow_mut(); - let zlib = zlib - .as_mut() - .ok_or_else(|| type_error("zlib not initialized"))?; + let zlib = zlib.as_mut().ok_or(ZlibError::NotInitialized)?; check((8..=15).contains(&window_bits), "invalid windowBits")?; check((-1..=9).contains(&level), "invalid level")?; @@ -348,13 +353,11 @@ pub fn op_zlib_init( #[op2(fast)] #[smi] -pub fn op_zlib_reset(#[cppgc] resource: &Zlib) -> Result { +pub fn op_zlib_reset(#[cppgc] resource: &Zlib) -> Result { let mut zlib = resource.inner.borrow_mut(); - let zlib = zlib - .as_mut() - .ok_or_else(|| type_error("zlib not initialized"))?; + let zlib = zlib.as_mut().ok_or(ZlibError::NotInitialized)?; - zlib.reset_stream()?; + zlib.reset_stream(); Ok(zlib.err) } @@ -362,12 +365,10 @@ pub fn op_zlib_reset(#[cppgc] resource: &Zlib) -> Result { #[op2(fast)] pub fn op_zlib_close_if_pending( #[cppgc] resource: &Zlib, -) -> Result<(), AnyError> { +) -> Result<(), ZlibError> { let pending_close = { let mut zlib = resource.inner.borrow_mut(); - let zlib = zlib - .as_mut() - .ok_or_else(|| type_error("zlib not initialized"))?; + let zlib = zlib.as_mut().ok_or(ZlibError::NotInitialized)?; zlib.write_in_progress = false; zlib.pending_close @@ -381,6 +382,15 @@ pub fn op_zlib_close_if_pending( Ok(()) } +#[op2(fast)] +#[smi] +pub fn op_zlib_crc32(#[buffer] data: &[u8], #[smi] value: u32) -> u32 { + // SAFETY: `data` is a valid buffer. + unsafe { + zlib::crc32(value as c_ulong, data.as_ptr(), data.len() as u32) as u32 + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/ext/node/ops/zlib/mode.rs b/ext/node/ops/zlib/mode.rs index 753300cc49c8f5..41565f9b1142df 100644 --- a/ext/node/ops/zlib/mode.rs +++ b/ext/node/ops/zlib/mode.rs @@ -1,19 +1,8 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -#[derive(Debug)] -pub enum Error { - BadArgument, -} - -impl std::fmt::Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Error::BadArgument => write!(f, "bad argument"), - } - } -} - -impl std::error::Error for Error {} +#[derive(Debug, thiserror::Error)] +#[error("bad argument")] +pub struct ModeError; macro_rules! repr_i32 { ($(#[$meta:meta])* $vis:vis enum $name:ident { @@ -25,12 +14,12 @@ macro_rules! repr_i32 { } impl core::convert::TryFrom for $name { - type Error = Error; + type Error = ModeError; fn try_from(v: i32) -> Result { match v { $(x if x == $name::$vname as i32 => Ok($name::$vname),)* - _ => Err(Error::BadArgument), + _ => Err(ModeError), } } } diff --git a/ext/node/polyfills/01_require.js b/ext/node/polyfills/01_require.js index 5b0980c310f6cf..df73cad6b7ddc2 100644 --- a/ext/node/polyfills/01_require.js +++ b/ext/node/polyfills/01_require.js @@ -11,6 +11,7 @@ import { op_require_can_parse_as_esm, op_require_init_paths, op_require_is_deno_dir_package, + op_require_is_maybe_cjs, op_require_is_request_relative, op_require_node_module_paths, op_require_package_imports_resolve, @@ -19,7 +20,6 @@ import { op_require_path_is_absolute, op_require_path_resolve, op_require_proxy_path, - op_require_read_closest_package_json, op_require_read_file, op_require_read_package_scope, op_require_real_path, @@ -523,17 +523,13 @@ function resolveExports( return; } - if (!parentPath) { - return false; - } - return op_require_resolve_exports( usesLocalNodeModulesDir, modulesPath, request, name, expansion, - parentPath, + parentPath ?? "", ) ?? false; } @@ -1065,22 +1061,38 @@ Module.prototype._compile = function (content, filename, format) { }; Module._extensions[".js"] = function (module, filename) { - const content = op_require_read_file(filename); - - let format; - if (StringPrototypeEndsWith(filename, ".js")) { - const pkg = op_require_read_closest_package_json(filename); - if (pkg?.type === "module") { - format = "module"; - } else if (pkg?.type === "commonjs") { - format = "commonjs"; - } - } else if (StringPrototypeEndsWith(filename, ".cjs")) { - format = "commonjs"; + // We don't define everything on Module.extensions in + // order to prevent probing for these files + if ( + StringPrototypeEndsWith(filename, ".js") || + StringPrototypeEndsWith(filename, ".ts") || + StringPrototypeEndsWith(filename, ".jsx") || + StringPrototypeEndsWith(filename, ".tsx") + ) { + return loadMaybeCjs(module, filename); + } else if (StringPrototypeEndsWith(filename, ".mts")) { + return loadESMFromCJS(module, filename); + } else if (StringPrototypeEndsWith(filename, ".cts")) { + return loadCjs(module, filename); + } else { + return loadMaybeCjs(module, filename); } +}; +Module._extensions[".cjs"] = loadCjs; +Module._extensions[".mjs"] = loadESMFromCJS; +Module._extensions[".wasm"] = loadESMFromCJS; + +function loadMaybeCjs(module, filename) { + const content = op_require_read_file(filename); + const format = op_require_is_maybe_cjs(filename) ? undefined : "module"; module._compile(content, filename, format); -}; +} + +function loadCjs(module, filename) { + const content = op_require_read_file(filename); + module._compile(content, filename, "commonjs"); +} function loadESMFromCJS(module, filename, code) { const namespace = op_import_sync( @@ -1091,10 +1103,6 @@ function loadESMFromCJS(module, filename, code) { module.exports = namespace; } -Module._extensions[".mjs"] = function (module, filename) { - loadESMFromCJS(module, filename); -}; - function stripBOM(content) { if (StringPrototypeCharCodeAt(content, 0) === 0xfeff) { content = StringPrototypeSlice(content, 1); @@ -1212,6 +1220,24 @@ function isBuiltin(moduleName) { !StringPrototypeStartsWith(moduleName, "internal/"); } +function getBuiltinModule(id) { + if (!isBuiltin(id)) { + return undefined; + } + + if (StringPrototypeStartsWith(id, "node:")) { + // Slice 'node:' prefix + id = StringPrototypeSlice(id, 5); + } + + const mod = loadNativeModule(id, id); + if (mod) { + return mod.exports; + } + + return undefined; +} + Module.isBuiltin = isBuiltin; Module.createRequire = createRequire; @@ -1291,6 +1317,8 @@ export function findSourceMap(_path) { return undefined; } +Module.findSourceMap = findSourceMap; + /** * @param {string | URL} _specifier * @param {string | URL} _parentUrl @@ -1304,7 +1332,7 @@ export function register(_specifier, _parentUrl, _options) { return undefined; } -export { builtinModules, createRequire, isBuiltin, Module }; +export { builtinModules, createRequire, getBuiltinModule, isBuiltin, Module }; export const _cache = Module._cache; export const _extensions = Module._extensions; export const _findPath = Module._findPath; diff --git a/ext/node/polyfills/_fs/_fs_common.ts b/ext/node/polyfills/_fs/_fs_common.ts index ac0bf5a5512c22..a29548bb364cc4 100644 --- a/ext/node/polyfills/_fs/_fs_common.ts +++ b/ext/node/polyfills/_fs/_fs_common.ts @@ -20,6 +20,7 @@ import { notImplemented, TextEncodings, } from "ext:deno_node/_utils.ts"; +import { type Buffer } from "node:buffer"; export type CallbackWithError = (err: ErrnoException | null) => void; diff --git a/ext/node/polyfills/_fs/_fs_copy.ts b/ext/node/polyfills/_fs/_fs_copy.ts index 2f8ddf4fcd25c2..0434bff4d10619 100644 --- a/ext/node/polyfills/_fs/_fs_copy.ts +++ b/ext/node/polyfills/_fs/_fs_copy.ts @@ -53,8 +53,9 @@ export function copyFile( }, (e) => { if (e instanceof Deno.errors.NotFound) { Deno.copyFile(srcStr, destStr).then(() => cb(null), cb); + } else { + cb(e); } - cb(e); }); } else { Deno.copyFile(srcStr, destStr).then(() => cb(null), cb); @@ -83,8 +84,9 @@ export function copyFileSync( } catch (e) { if (e instanceof Deno.errors.NotFound) { Deno.copyFileSync(srcStr, destStr); + } else { + throw e; } - throw e; } } else { Deno.copyFileSync(srcStr, destStr); diff --git a/ext/node/polyfills/_fs/_fs_fstat.ts b/ext/node/polyfills/_fs/_fs_fstat.ts index c1722487eb3160..1a845dfff4a453 100644 --- a/ext/node/polyfills/_fs/_fs_fstat.ts +++ b/ext/node/polyfills/_fs/_fs_fstat.ts @@ -63,3 +63,24 @@ export function fstatSync( const origin = new FsFile(fd, Symbol.for("Deno.internal.FsFile")).statSync(); return CFISBIS(origin, options?.bigint || false); } + +export function fstatPromise(fd: number): Promise; +export function fstatPromise( + fd: number, + options: { bigint: false }, +): Promise; +export function fstatPromise( + fd: number, + options: { bigint: true }, +): Promise; +export function fstatPromise( + fd: number, + options?: statOptions, +): Stats | BigIntStats { + return new Promise((resolve, reject) => { + fstat(fd, options, (err, stats) => { + if (err) reject(err); + else resolve(stats); + }); + }); +} diff --git a/ext/node/polyfills/_fs/_fs_open.ts b/ext/node/polyfills/_fs/_fs_open.ts index 8bd989790b9deb..31ca4bb6195857 100644 --- a/ext/node/polyfills/_fs/_fs_open.ts +++ b/ext/node/polyfills/_fs/_fs_open.ts @@ -147,8 +147,8 @@ export function open( export function openPromise( path: string | Buffer | URL, - flags?: openFlags = "r", - mode? = 0o666, + flags: openFlags = "r", + mode = 0o666, ): Promise { return new Promise((resolve, reject) => { open(path, flags, mode, (err, fd) => { diff --git a/ext/node/polyfills/_fs/_fs_read.ts b/ext/node/polyfills/_fs/_fs_read.ts index dec3a8bbdfb2ae..df4f5e375dd00f 100644 --- a/ext/node/polyfills/_fs/_fs_read.ts +++ b/ext/node/polyfills/_fs/_fs_read.ts @@ -173,7 +173,7 @@ export function readSync( validateBuffer(buffer); if (length == null) { - length = 0; + length = buffer.byteLength; } if (typeof offsetOrOpt === "number") { diff --git a/ext/node/polyfills/_fs/_fs_readFile.ts b/ext/node/polyfills/_fs/_fs_readFile.ts index 0f05ee1678cec2..029e57c5029e0c 100644 --- a/ext/node/polyfills/_fs/_fs_readFile.ts +++ b/ext/node/polyfills/_fs/_fs_readFile.ts @@ -19,6 +19,7 @@ import { TextEncodings, } from "ext:deno_node/_utils.ts"; import { FsFile } from "ext:deno_fs/30_fs.js"; +import { denoErrorToNodeError } from "ext:deno_node/internal/errors.ts"; function maybeDecode(data: Uint8Array, encoding: TextEncodings): string; function maybeDecode( @@ -87,7 +88,7 @@ export function readFile( } const buffer = maybeDecode(data, encoding); (cb as BinaryCallback)(null, buffer); - }, (err) => cb && cb(err)); + }, (err) => cb && cb(denoErrorToNodeError(err, { path, syscall: "open" }))); } } @@ -117,7 +118,12 @@ export function readFileSync( opt?: FileOptionsArgument, ): string | Buffer { path = path instanceof URL ? pathFromURL(path) : path; - const data = Deno.readFileSync(path); + let data; + try { + data = Deno.readFileSync(path); + } catch (err) { + throw denoErrorToNodeError(err, { path, syscall: "open" }); + } const encoding = getEncoding(opt); if (encoding && encoding !== "binary") { const text = maybeDecode(data, encoding); diff --git a/ext/node/polyfills/_fs/_fs_readlink.ts b/ext/node/polyfills/_fs/_fs_readlink.ts index 5f231279826fdf..08bea843faf585 100644 --- a/ext/node/polyfills/_fs/_fs_readlink.ts +++ b/ext/node/polyfills/_fs/_fs_readlink.ts @@ -4,13 +4,10 @@ // deno-lint-ignore-file prefer-primordials import { TextEncoder } from "ext:deno_web/08_text_encoding.js"; -import { - intoCallbackAPIWithIntercept, - MaybeEmpty, - notImplemented, -} from "ext:deno_node/_utils.ts"; +import { MaybeEmpty, notImplemented } from "ext:deno_node/_utils.ts"; import { pathFromURL } from "ext:deno_web/00_infra.js"; import { promisify } from "ext:deno_node/internal/util.mjs"; +import { denoErrorToNodeError } from "ext:deno_node/internal/errors.ts"; type ReadlinkCallback = ( err: MaybeEmpty, @@ -69,12 +66,17 @@ export function readlink( const encoding = getEncoding(optOrCallback); - intoCallbackAPIWithIntercept( - Deno.readLink, - (data: string): string | Uint8Array => maybeEncode(data, encoding), - cb, - path, - ); + Deno.readLink(path).then((data: string) => { + const res = maybeEncode(data, encoding); + if (cb) cb(null, res); + }, (err: Error) => { + if (cb) { + (cb as (e: Error) => void)(denoErrorToNodeError(err, { + syscall: "readlink", + path, + })); + } + }); } export const readlinkPromise = promisify(readlink) as ( @@ -88,5 +90,12 @@ export function readlinkSync( ): string | Uint8Array { path = path instanceof URL ? pathFromURL(path) : path; - return maybeEncode(Deno.readLinkSync(path), getEncoding(opt)); + try { + return maybeEncode(Deno.readLinkSync(path), getEncoding(opt)); + } catch (error) { + throw denoErrorToNodeError(error, { + syscall: "readlink", + path, + }); + } } diff --git a/ext/node/polyfills/_fs/_fs_readv.ts b/ext/node/polyfills/_fs/_fs_readv.ts index 384f5e319ae96c..2259f029aebd7f 100644 --- a/ext/node/polyfills/_fs/_fs_readv.ts +++ b/ext/node/polyfills/_fs/_fs_readv.ts @@ -15,6 +15,7 @@ import { maybeCallback } from "ext:deno_node/_fs/_fs_common.ts"; import { validateInteger } from "ext:deno_node/internal/validators.mjs"; import * as io from "ext:deno_io/12_io.js"; import { op_fs_seek_async, op_fs_seek_sync } from "ext:core/ops"; +import process from "node:process"; type Callback = ( err: ErrnoException | null, diff --git a/ext/node/polyfills/_fs/_fs_stat.ts b/ext/node/polyfills/_fs/_fs_stat.ts index c4ed82d577a926..f26474668665b9 100644 --- a/ext/node/polyfills/_fs/_fs_stat.ts +++ b/ext/node/polyfills/_fs/_fs_stat.ts @@ -6,6 +6,7 @@ import { denoErrorToNodeError } from "ext:deno_node/internal/errors.ts"; import { promisify } from "ext:deno_node/internal/util.mjs"; import { primordials } from "ext:core/mod.js"; +import { getValidatedPath } from "ext:deno_node/internal/fs/utils.mjs"; const { ObjectCreate, ObjectAssign } = primordials; @@ -290,8 +291,8 @@ export function convertFileInfoToStats(origin: Deno.FileInfo): Stats { isFIFO: () => false, isCharacterDevice: () => false, isSocket: () => false, - ctime: origin.mtime, - ctimeMs: origin.mtime?.getTime() || null, + ctime: origin.ctime, + ctimeMs: origin.ctime?.getTime() || null, }); return stats; @@ -336,9 +337,9 @@ export function convertFileInfoToBigIntStats( isFIFO: () => false, isCharacterDevice: () => false, isSocket: () => false, - ctime: origin.mtime, - ctimeMs: origin.mtime ? BigInt(origin.mtime.getTime()) : null, - ctimeNs: origin.mtime ? BigInt(origin.mtime.getTime()) * 1000000n : null, + ctime: origin.ctime, + ctimeMs: origin.ctime ? BigInt(origin.ctime.getTime()) : null, + ctimeNs: origin.ctime ? BigInt(origin.ctime.getTime()) * 1000000n : null, }); return stats; } @@ -379,11 +380,15 @@ export function stat( ? optionsOrCallback : { bigint: false }; + path = getValidatedPath(path).toString(); if (!callback) throw new Error("No callback function supplied"); Deno.stat(path).then( (stat) => callback(null, CFISBIS(stat, options.bigint)), - (err) => callback(denoErrorToNodeError(err, { syscall: "stat" })), + (err) => + callback( + denoErrorToNodeError(err, { syscall: "stat", path: getPathname(path) }), + ), ); } @@ -406,6 +411,8 @@ export function statSync( path: string | URL, options: statOptions = { bigint: false, throwIfNoEntry: true }, ): Stats | BigIntStats | undefined { + path = getValidatedPath(path).toString(); + try { const origin = Deno.statSync(path); return CFISBIS(origin, options.bigint); @@ -417,9 +424,16 @@ export function statSync( return; } if (err instanceof Error) { - throw denoErrorToNodeError(err, { syscall: "stat" }); + throw denoErrorToNodeError(err, { + syscall: "stat", + path: getPathname(path), + }); } else { throw err; } } } + +function getPathname(path: string | URL) { + return typeof path === "string" ? path : path.pathname; +} diff --git a/ext/node/polyfills/_next_tick.ts b/ext/node/polyfills/_next_tick.ts index 5ee27728d0d25d..af306a29c83c0c 100644 --- a/ext/node/polyfills/_next_tick.ts +++ b/ext/node/polyfills/_next_tick.ts @@ -62,6 +62,8 @@ export function processTicksAndRejections() { callback(...args); } } + } catch (e) { + reportError(e); } finally { // FIXME(bartlomieju): Deno currently doesn't support async hooks // if (destroyHooksExist()) @@ -87,8 +89,7 @@ export function runNextTicks() { // runMicrotasks(); // if (!hasTickScheduled() && !hasRejectionToWarn()) // return; - if (!core.hasTickScheduled()) { - core.runMicrotasks(); + if (queue.isEmpty() || !core.hasTickScheduled()) { return true; } diff --git a/ext/node/polyfills/_process/streams.mjs b/ext/node/polyfills/_process/streams.mjs index 19c1c9c18285ac..3573956c9daf21 100644 --- a/ext/node/polyfills/_process/streams.mjs +++ b/ext/node/polyfills/_process/streams.mjs @@ -63,13 +63,22 @@ export function createWritableStdioStream(writer, name, warmup = false) { stream.destroySoon = stream.destroy; stream._isStdio = true; stream.once("close", () => writer?.close()); + + // We cannot call `writer?.isTerminal()` eagerly here + let getIsTTY = () => writer?.isTerminal(); + const getColumns = () => + stream._columns || + (writer?.isTerminal() ? Deno.consoleSize?.().columns : undefined); + ObjectDefineProperties(stream, { columns: { __proto__: null, enumerable: true, configurable: true, - get: () => - writer?.isTerminal() ? Deno.consoleSize?.().columns : undefined, + get: () => getColumns(), + set: (value) => { + stream._columns = value; + }, }, rows: { __proto__: null, @@ -81,7 +90,11 @@ export function createWritableStdioStream(writer, name, warmup = false) { __proto__: null, enumerable: true, configurable: true, - get: () => writer?.isTerminal(), + // Allow users to overwrite it + get: () => getIsTTY(), + set: (value) => { + getIsTTY = () => value; + }, }, getWindowSize: { __proto__: null, diff --git a/ext/node/polyfills/_tls_wrap.ts b/ext/node/polyfills/_tls_wrap.ts index a614b45df0a8d2..4c7424a3287f16 100644 --- a/ext/node/polyfills/_tls_wrap.ts +++ b/ext/node/polyfills/_tls_wrap.ts @@ -68,6 +68,7 @@ export class TLSSocket extends net.Socket { secureConnecting: boolean; _SNICallback: any; servername: string | null; + alpnProtocol: string | boolean | null; alpnProtocols: string[] | null; authorized: boolean; authorizationError: any; @@ -114,6 +115,7 @@ export class TLSSocket extends net.Socket { this.secureConnecting = true; this._SNICallback = null; this.servername = null; + this.alpnProtocol = null; this.alpnProtocols = tlsOptions.ALPNProtocols; this.authorized = false; this.authorizationError = null; @@ -146,12 +148,36 @@ export class TLSSocket extends net.Socket { : new TCP(TCPConstants.SOCKET); } + const { promise, resolve } = Promise.withResolvers(); + // Patches `afterConnect` hook to replace TCP conn with TLS conn const afterConnect = handle.afterConnect; handle.afterConnect = async (req: any, status: number) => { + options.hostname ??= undefined; // coerce to undefined if null, startTls expects hostname to be undefined + try { const conn = await Deno.startTls(handle[kStreamBaseField], options); + try { + const hs = await conn.handshake(); + if (hs.alpnProtocol) { + tlssock.alpnProtocol = hs.alpnProtocol; + } else { + tlssock.alpnProtocol = false; + } + } catch { + // Don't interrupt "secure" event to let the first read/write + // operation emit the error. + } + + // Assign the TLS connection to the handle and resume reading. handle[kStreamBaseField] = conn; + handle.upgrading = false; + if (!handle.pauseOnCreate) { + handle.readStart(); + } + + resolve(); + tlssock.emit("secure"); tlssock.removeListener("end", onConnectEnd); } catch { @@ -160,6 +186,7 @@ export class TLSSocket extends net.Socket { return afterConnect.call(handle, req, status); }; + handle.upgrading = promise; (handle as any).verifyError = function () { return null; // Never fails, rejectUnauthorized is always true in Deno. }; @@ -269,6 +296,7 @@ export class ServerImpl extends EventEmitter { // Creates TCP handle and socket directly from Deno.TlsConn. // This works as TLS socket. We don't use TLSSocket class for doing // this because Deno.startTls only supports client side tcp connection. + // TODO(@satyarohith): set TLSSocket.alpnProtocol when we use TLSSocket class. const handle = new TCP(TCPConstants.SOCKET, await listener.accept()); const socket = new net.Socket({ handle }); this.emit("secureConnection", socket); diff --git a/ext/node/polyfills/_utils.ts b/ext/node/polyfills/_utils.ts index b50c113e14f3eb..79d84e00f0c874 100644 --- a/ext/node/polyfills/_utils.ts +++ b/ext/node/polyfills/_utils.ts @@ -17,6 +17,7 @@ const { import { TextDecoder, TextEncoder } from "ext:deno_web/08_text_encoding.js"; import { errorMap } from "ext:deno_node/internal_binding/uv.ts"; import { codes } from "ext:deno_node/internal/error_codes.ts"; +import { ERR_NOT_IMPLEMENTED } from "ext:deno_node/internal/errors.ts"; export type BinaryEncodings = "binary"; @@ -34,8 +35,7 @@ export type TextEncodings = export type Encodings = BinaryEncodings | TextEncodings; export function notImplemented(msg: string): never { - const message = msg ? `Not implemented: ${msg}` : "Not implemented"; - throw new Error(message); + throw new ERR_NOT_IMPLEMENTED(msg); } export function warnNotImplemented(msg?: string) { diff --git a/ext/node/polyfills/_zlib.mjs b/ext/node/polyfills/_zlib.mjs index 851bd602f2f624..07fc440ef545e5 100644 --- a/ext/node/polyfills/_zlib.mjs +++ b/ext/node/polyfills/_zlib.mjs @@ -14,6 +14,7 @@ import { nextTick } from "ext:deno_node/_next_tick.ts"; import { isAnyArrayBuffer, isArrayBufferView, + isUint8Array, } from "ext:deno_node/internal/util/types.ts"; var kRangeErrorMessage = "Cannot create final Buffer. It would be larger " + @@ -158,6 +159,12 @@ export const inflateRawSync = function (buffer, opts) { function sanitizeInput(input) { if (typeof input === "string") input = Buffer.from(input); + if (isArrayBufferView(input) && !isUint8Array(input)) { + input = Buffer.from(input.buffer, input.byteOffset, input.byteLength); + } else if (isAnyArrayBuffer(input)) { + input = Buffer.from(input); + } + if ( !Buffer.isBuffer(input) && (input.buffer && !input.buffer.constructor === ArrayBuffer) diff --git a/ext/node/polyfills/buffer.ts b/ext/node/polyfills/buffer.ts index 8986cf53d7f77a..efe3b07a97a451 100644 --- a/ext/node/polyfills/buffer.ts +++ b/ext/node/polyfills/buffer.ts @@ -13,4 +13,5 @@ export { kMaxLength, kStringMaxLength, SlowBuffer, + transcode, } from "ext:deno_node/internal/buffer.mjs"; diff --git a/ext/node/polyfills/child_process.ts b/ext/node/polyfills/child_process.ts index f77a430c2cb352..eda718ff3480c3 100644 --- a/ext/node/polyfills/child_process.ts +++ b/ext/node/polyfills/child_process.ts @@ -10,7 +10,6 @@ import { internals } from "ext:core/mod.js"; import { op_bootstrap_unstable_args, op_node_child_ipc_pipe, - op_npm_process_state, } from "ext:core/ops"; import { @@ -54,6 +53,7 @@ import { convertToValidSignal, kEmptyObject, } from "ext:deno_node/internal/util.mjs"; +import { kNeedsNpmProcessState } from "ext:runtime/40_process.js"; const MAX_BUFFER = 1024 * 1024; @@ -132,6 +132,8 @@ export function fork( rm = 2; } execArgv.splice(index, rm); + } else if (flag.startsWith("--no-warnings")) { + execArgv[index] = "--quiet"; } else { index++; } @@ -168,9 +170,8 @@ export function fork( options.execPath = options.execPath || Deno.execPath(); options.shell = false; - Object.assign(options.env ??= {}, { - DENO_DONT_USE_INTERNAL_NODE_COMPAT_STATE: op_npm_process_state(), - }); + // deno-lint-ignore no-explicit-any + (options as any)[kNeedsNpmProcessState] = true; return spawn(options.execPath, args, options); } diff --git a/ext/node/polyfills/fs.ts b/ext/node/polyfills/fs.ts index 7a3cf4e67f2d52..cbdc36afe55652 100644 --- a/ext/node/polyfills/fs.ts +++ b/ext/node/polyfills/fs.ts @@ -23,7 +23,7 @@ import Dir from "ext:deno_node/_fs/_fs_dir.ts"; import Dirent from "ext:deno_node/_fs/_fs_dirent.ts"; import { exists, existsSync } from "ext:deno_node/_fs/_fs_exists.ts"; import { fdatasync, fdatasyncSync } from "ext:deno_node/_fs/_fs_fdatasync.ts"; -import { fstat, fstatSync } from "ext:deno_node/_fs/_fs_fstat.ts"; +import { fstat, fstatPromise, fstatSync } from "ext:deno_node/_fs/_fs_fstat.ts"; import { fsync, fsyncSync } from "ext:deno_node/_fs/_fs_fsync.ts"; import { ftruncate, ftruncateSync } from "ext:deno_node/_fs/_fs_ftruncate.ts"; import { futimes, futimesSync } from "ext:deno_node/_fs/_fs_futimes.ts"; @@ -174,6 +174,7 @@ const promises = { lstat: lstatPromise, stat: statPromise, statfs: statfsPromise, + fstat: fstatPromise, link: linkPromise, unlink: unlinkPromise, chmod: chmodPromise, diff --git a/ext/node/polyfills/fs/promises.ts b/ext/node/polyfills/fs/promises.ts index 3e5329dbbe932f..a5125dac8dc9d7 100644 --- a/ext/node/polyfills/fs/promises.ts +++ b/ext/node/polyfills/fs/promises.ts @@ -16,6 +16,7 @@ export const readlink = fsPromises.readlink; export const symlink = fsPromises.symlink; export const lstat = fsPromises.lstat; export const stat = fsPromises.stat; +export const fstat = fsPromises.fstat; export const link = fsPromises.link; export const unlink = fsPromises.unlink; export const chmod = fsPromises.chmod; diff --git a/ext/node/polyfills/http.ts b/ext/node/polyfills/http.ts index f3f6f86ed85811..948a3527bdc8f0 100644 --- a/ext/node/polyfills/http.ts +++ b/ext/node/polyfills/http.ts @@ -34,6 +34,7 @@ import { finished, Readable as NodeReadable, Writable as NodeWritable, + WritableOptions as NodeWritableOptions, } from "node:stream"; import { kUniqueHeaders, @@ -66,12 +67,13 @@ import { headersEntries } from "ext:deno_fetch/20_headers.js"; import { timerId } from "ext:deno_web/03_abort_signal.js"; import { clearTimeout as webClearTimeout } from "ext:deno_web/02_timers.js"; import { resourceForReadableStream } from "ext:deno_web/06_streams.js"; -import { TcpConn } from "ext:deno_net/01_net.js"; +import { UpgradedConn } from "ext:deno_net/01_net.js"; import { STATUS_CODES } from "node:_http_server"; import { methods as METHODS } from "node:_http_common"; +import { deprecate } from "node:util"; const { internalRidSymbol } = core; -const { ArrayIsArray } = primordials; +const { ArrayIsArray, StringPrototypeToLowerCase } = primordials; type Chunk = string | Buffer | Uint8Array; @@ -516,7 +518,7 @@ class ClientRequest extends OutgoingMessage { ); assert(typeof res.remoteAddrIp !== "undefined"); assert(typeof res.remoteAddrIp !== "undefined"); - const conn = new TcpConn( + const conn = new UpgradedConn( upgradeRid, { transport: "tcp", @@ -1183,49 +1185,95 @@ function onError(self, error, cb) { } } -export class ServerResponse extends NodeWritable { - statusCode = 200; - statusMessage?: string = undefined; - #headers: Record = { __proto__: null }; - #hasNonStringHeaders: boolean = false; - #readable: ReadableStream; - override writable = true; - // used by `npm:on-finished` - finished = false; - headersSent = false; - #resolve: (value: Response | PromiseLike) => void; +export type ServerResponse = { + statusCode: number; + statusMessage?: string; + + _headers: Record; + _hasNonStringHeaders: boolean; + + _readable: ReadableStream; + finished: boolean; + headersSent: boolean; + _resolve: (value: Response | PromiseLike) => void; + // deno-lint-ignore no-explicit-any + _socketOverride: any | null; // deno-lint-ignore no-explicit-any - #socketOverride: any | null = null; + socket: any | null; - static #enqueue(controller: ReadableStreamDefaultController, chunk: Chunk) { - try { - if (typeof chunk === "string") { - controller.enqueue(ENCODER.encode(chunk)); - } else { - controller.enqueue(chunk); - } - } catch (_) { - // The stream might have been closed. Ignore the error. - } - } + setHeader(name: string, value: string | string[]): void; + appendHeader(name: string, value: string | string[]): void; + getHeader(name: string): string | string[]; + removeHeader(name: string): void; + getHeaderNames(): string[]; + getHeaders(): Record; + hasHeader(name: string): boolean; - /** Returns true if the response body should be null with the given - * http status code */ - static #bodyShouldBeNull(status: number) { - return status === 101 || status === 204 || status === 205 || status === 304; - } + writeHead( + status: number, + statusMessage?: string, + headers?: + | Record + | Array<[string, string]>, + ): void; + writeHead( + status: number, + headers?: + | Record + | Array<[string, string]>, + ): void; - constructor( + _ensureHeaders(singleChunk?: Chunk): void; + + respond(final: boolean, singleChunk?: Chunk): void; + // deno-lint-ignore no-explicit-any + end(chunk?: any, encoding?: any, cb?: any): void; + + flushHeaders(): void; + _implicitHeader(): void; + + // Undocumented field used by `npm:light-my-request`. + _header: string; + + assignSocket(socket): void; + detachSocket(socket): void; +} & { -readonly [K in keyof NodeWritable]: NodeWritable[K] }; + +type ServerResponseStatic = { + new ( resolve: (value: Response | PromiseLike) => void, socket: FakeSocket, - ) { - let controller: ReadableByteStreamController; - const readable = new ReadableStream({ - start(c) { - controller = c as ReadableByteStreamController; - }, - }); - super({ + ): ServerResponse; + _enqueue(controller: ReadableStreamDefaultController, chunk: Chunk): void; + _bodyShouldBeNull(statusCode: number): boolean; +}; + +export const ServerResponse = function ( + this: ServerResponse, + resolve: (value: Response | PromiseLike) => void, + socket: FakeSocket, +) { + this.statusCode = 200; + this.statusMessage = undefined; + this._headers = { __proto__: null }; + this._hasNonStringHeaders = false; + this.writable = true; + + // used by `npm:on-finished` + this.finished = false; + this.headersSent = false; + this._socketOverride = null; + + let controller: ReadableByteStreamController; + const readable = new ReadableStream({ + start(c) { + controller = c as ReadableByteStreamController; + }, + }); + + NodeWritable.call( + this, + { autoDestroy: true, defaultEncoding: "utf-8", emitClose: true, @@ -1234,16 +1282,16 @@ export class ServerResponse extends NodeWritable { write: (chunk, encoding, cb) => { // Writes chunks are directly written to the socket if // one is assigned via assignSocket() - if (this.#socketOverride && this.#socketOverride.writable) { - this.#socketOverride.write(chunk, encoding); + if (this._socketOverride && this._socketOverride.writable) { + this._socketOverride.write(chunk, encoding); return cb(); } if (!this.headersSent) { - ServerResponse.#enqueue(controller, chunk); + ServerResponse._enqueue(controller, chunk); this.respond(false); return cb(); } - ServerResponse.#enqueue(controller, chunk); + ServerResponse._enqueue(controller, chunk); return cb(); }, final: (cb) => { @@ -1259,192 +1307,269 @@ export class ServerResponse extends NodeWritable { } return cb(null); }, - }); - this.#readable = readable; - this.#resolve = resolve; - this.socket = socket; + } satisfies NodeWritableOptions, + ); + + this._readable = readable; + this._resolve = resolve; + this.socket = socket; + + this._header = ""; +} as unknown as ServerResponseStatic; + +Object.setPrototypeOf(ServerResponse.prototype, NodeWritable.prototype); +Object.setPrototypeOf(ServerResponse, NodeWritable); + +ServerResponse._enqueue = function ( + this: ServerResponse, + controller: ReadableStreamDefaultController, + chunk: Chunk, +) { + try { + if (typeof chunk === "string") { + controller.enqueue(ENCODER.encode(chunk)); + } else { + controller.enqueue(chunk); + } + } catch (_) { + // The stream might have been closed. Ignore the error. } +}; - setHeader(name: string, value: string | string[]) { - if (Array.isArray(value)) { - this.#hasNonStringHeaders = true; - } - this.#headers[name] = value; - return this; +/** Returns true if the response body should be null with the given + * http status code */ +ServerResponse._bodyShouldBeNull = function ( + this: ServerResponse, + status: number, +) { + return status === 101 || status === 204 || status === 205 || status === 304; +}; + +ServerResponse.prototype.setHeader = function ( + this: ServerResponse, + name: string, + value: string | string[], +) { + if (Array.isArray(value)) { + this._hasNonStringHeaders = true; } + this._headers[StringPrototypeToLowerCase(name)] = value; + return this; +}; - appendHeader(name: string, value: string | string[]) { - if (this.#headers[name] === undefined) { - if (Array.isArray(value)) this.#hasNonStringHeaders = true; - this.#headers[name] = value; +ServerResponse.prototype.appendHeader = function ( + this: ServerResponse, + name: string, + value: string | string[], +) { + const key = StringPrototypeToLowerCase(name); + if (this._headers[key] === undefined) { + if (Array.isArray(value)) this._hasNonStringHeaders = true; + this._headers[key] = value; + } else { + this._hasNonStringHeaders = true; + if (!Array.isArray(this._headers[key])) { + this._headers[key] = [this._headers[key]]; + } + const header = this._headers[key]; + if (Array.isArray(value)) { + header.push(...value); } else { - this.#hasNonStringHeaders = true; - if (!Array.isArray(this.#headers[name])) { - this.#headers[name] = [this.#headers[name]]; - } - const header = this.#headers[name]; - if (Array.isArray(value)) { - header.push(...value); - } else { - header.push(value); - } + header.push(value); } - return this; } + return this; +}; - getHeader(name: string) { - return this.#headers[name]; - } - removeHeader(name: string) { - delete this.#headers[name]; - } - getHeaderNames() { - return Object.keys(this.#headers); - } - getHeaders(): Record { - // @ts-ignore Ignore null __proto__ - return { __proto__: null, ...this.#headers }; - } - hasHeader(name: string) { - return Object.hasOwn(this.#headers, name); - } +ServerResponse.prototype.getHeader = function ( + this: ServerResponse, + name: string, +) { + return this._headers[StringPrototypeToLowerCase(name)]; +}; - writeHead( - status: number, - statusMessage?: string, - headers?: - | Record - | Array<[string, string]>, - ): this; - writeHead( - status: number, - headers?: - | Record - | Array<[string, string]>, - ): this; - writeHead( - status: number, - statusMessageOrHeaders?: - | string - | Record - | Array<[string, string]>, - maybeHeaders?: - | Record - | Array<[string, string]>, - ): this { - this.statusCode = status; - - let headers = null; - if (typeof statusMessageOrHeaders === "string") { - this.statusMessage = statusMessageOrHeaders; - if (maybeHeaders !== undefined) { - headers = maybeHeaders; - } - } else if (statusMessageOrHeaders !== undefined) { - headers = statusMessageOrHeaders; - } +ServerResponse.prototype.removeHeader = function ( + this: ServerResponse, + name: string, +) { + delete this._headers[StringPrototypeToLowerCase(name)]; +}; - if (headers !== null) { - if (ArrayIsArray(headers)) { - headers = headers as Array<[string, string]>; - for (let i = 0; i < headers.length; i++) { - this.appendHeader(headers[i][0], headers[i][1]); - } - } else { - headers = headers as Record; - for (const k in headers) { - if (Object.hasOwn(headers, k)) { - this.setHeader(k, headers[k]); - } +ServerResponse.prototype.getHeaderNames = function (this: ServerResponse) { + return Object.keys(this._headers); +}; + +ServerResponse.prototype.getHeaders = function ( + this: ServerResponse, +): Record { + return { __proto__: null, ...this._headers }; +}; + +ServerResponse.prototype.hasHeader = function ( + this: ServerResponse, + name: string, +) { + return Object.hasOwn(this._headers, StringPrototypeToLowerCase(name)); +}; + +ServerResponse.prototype.writeHead = function ( + this: ServerResponse, + status: number, + statusMessageOrHeaders?: + | string + | Record + | Array<[string, string]>, + maybeHeaders?: + | Record + | Array<[string, string]>, +) { + this.statusCode = status; + + let headers = null; + if (typeof statusMessageOrHeaders === "string") { + this.statusMessage = statusMessageOrHeaders; + if (maybeHeaders !== undefined) { + headers = maybeHeaders; + } + } else if (statusMessageOrHeaders !== undefined) { + headers = statusMessageOrHeaders; + } + + if (headers !== null) { + if (ArrayIsArray(headers)) { + headers = headers as Array<[string, string]>; + for (let i = 0; i < headers.length; i++) { + this.appendHeader(headers[i][0], headers[i][1]); + } + } else { + headers = headers as Record; + for (const k in headers) { + if (Object.hasOwn(headers, k)) { + this.setHeader(k, headers[k]); } } } + } - return this; + return this; +}; + +ServerResponse.prototype._ensureHeaders = function ( + this: ServerResponse, + singleChunk?: Chunk, +) { + if (this.statusCode === 200 && this.statusMessage === undefined) { + this.statusMessage = "OK"; } + if (typeof singleChunk === "string" && !this.hasHeader("content-type")) { + this.setHeader("content-type", "text/plain;charset=UTF-8"); + } +}; - #ensureHeaders(singleChunk?: Chunk) { - if (this.statusCode === 200 && this.statusMessage === undefined) { - this.statusMessage = "OK"; - } - if ( - typeof singleChunk === "string" && - !this.hasHeader("content-type") - ) { - this.setHeader("content-type", "text/plain;charset=UTF-8"); - } - } - - respond(final: boolean, singleChunk?: Chunk) { - this.headersSent = true; - this.#ensureHeaders(singleChunk); - let body = singleChunk ?? (final ? null : this.#readable); - if (ServerResponse.#bodyShouldBeNull(this.statusCode)) { - body = null; - } - let headers: Record | [string, string][] = this - .#headers as Record; - if (this.#hasNonStringHeaders) { - headers = []; - // Guard is not needed as this is a null prototype object. - // deno-lint-ignore guard-for-in - for (const key in this.#headers) { - const entry = this.#headers[key]; - if (Array.isArray(entry)) { - for (const value of entry) { - headers.push([key, value]); - } - } else { - headers.push([key, entry]); +ServerResponse.prototype.respond = function ( + this: ServerResponse, + final: boolean, + singleChunk?: Chunk, +) { + this.headersSent = true; + this._ensureHeaders(singleChunk); + let body = singleChunk ?? (final ? null : this._readable); + if (ServerResponse._bodyShouldBeNull(this.statusCode)) { + body = null; + } + let headers: Record | [string, string][] = this + ._headers as Record; + if (this._hasNonStringHeaders) { + headers = []; + // Guard is not needed as this is a null prototype object. + // deno-lint-ignore guard-for-in + for (const key in this._headers) { + const entry = this._headers[key]; + if (Array.isArray(entry)) { + for (const value of entry) { + headers.push([key, value]); } + } else { + headers.push([key, entry]); } } - this.#resolve( - new Response(body, { - headers, - status: this.statusCode, - statusText: this.statusMessage, - }), - ); } + this._resolve( + new Response(body, { + headers, + status: this.statusCode, + statusText: this.statusMessage, + }), + ); +}; +ServerResponse.prototype.end = function ( + this: ServerResponse, // deno-lint-ignore no-explicit-any - override end(chunk?: any, encoding?: any, cb?: any): this { - this.finished = true; - if (!chunk && "transfer-encoding" in this.#headers) { - // FIXME(bnoordhuis) Node sends a zero length chunked body instead, i.e., - // the trailing "0\r\n", but respondWith() just hangs when I try that. - this.#headers["content-length"] = "0"; - delete this.#headers["transfer-encoding"]; - } + chunk?: any, + // deno-lint-ignore no-explicit-any + encoding?: any, + // deno-lint-ignore no-explicit-any + cb?: any, +) { + this.finished = true; + if (!chunk && "transfer-encoding" in this._headers) { + // FIXME(bnoordhuis) Node sends a zero length chunked body instead, i.e., + // the trailing "0\r\n", but respondWith() just hangs when I try that. + this._headers["content-length"] = "0"; + delete this._headers["transfer-encoding"]; + } + + // @ts-expect-error The signature for cb is stricter than the one implemented here + NodeWritable.prototype.end.call(this, chunk, encoding, cb); +}; - // @ts-expect-error The signature for cb is stricter than the one implemented here - return super.end(chunk, encoding, cb); - } +ServerResponse.prototype.flushHeaders = function (this: ServerResponse) { + // no-op +}; - flushHeaders() { - // no-op - } +// Undocumented API used by `npm:compression`. +ServerResponse.prototype._implicitHeader = function (this: ServerResponse) { + this.writeHead(this.statusCode); +}; - // Undocumented API used by `npm:compression`. - _implicitHeader() { - this.writeHead(this.statusCode); +ServerResponse.prototype.assignSocket = function ( + this: ServerResponse, + socket, +) { + if (socket._httpMessage) { + throw new ERR_HTTP_SOCKET_ASSIGNED(); } + socket._httpMessage = this; + this._socketOverride = socket; +}; - assignSocket(socket) { - if (socket._httpMessage) { - throw new ERR_HTTP_SOCKET_ASSIGNED(); - } - socket._httpMessage = this; - this.#socketOverride = socket; - } +ServerResponse.prototype.detachSocket = function ( + this: ServerResponse, + socket, +) { + assert(socket._httpMessage === this); + socket._httpMessage = null; + this._socketOverride = null; +}; - detachSocket(socket) { - assert(socket._httpMessage === this); - socket._httpMessage = null; - this.#socketOverride = null; - } -} +Object.defineProperty(ServerResponse.prototype, "connection", { + get: deprecate( + function (this: ServerResponse) { + return this._socketOverride; + }, + "ServerResponse.prototype.connection is deprecated", + "DEP0066", + ), + set: deprecate( + // deno-lint-ignore no-explicit-any + function (this: ServerResponse, socket: any) { + this._socketOverride = socket; + }, + "ServerResponse.prototype.connection is deprecated", + "DEP0066", + ), +}); // TODO(@AaronO): optimize export class IncomingMessageForServer extends NodeReadable { @@ -1677,6 +1802,8 @@ export class ServerImpl extends EventEmitter { this.#server.ref(); } this.#unref = false; + + return this; } unref() { @@ -1684,6 +1811,8 @@ export class ServerImpl extends EventEmitter { this.#server.unref(); } this.#unref = true; + + return this; } close(cb?: (err?: Error) => void): this { diff --git a/ext/node/polyfills/http2.ts b/ext/node/polyfills/http2.ts index a9ced2bd9e9261..1b3f74f6f66ffa 100644 --- a/ext/node/polyfills/http2.ts +++ b/ext/node/polyfills/http2.ts @@ -479,13 +479,13 @@ export class ClientHttp2Session extends Http2Session { socket.on("error", socketOnError); socket.on("close", socketOnClose); + + socket[kHandle].pauseOnCreate = true; const connPromise = new Promise((resolve) => { const eventName = url.startsWith("https") ? "secureConnect" : "connect"; socket.once(eventName, () => { const rid = socket[kHandle][kStreamBaseField][internalRidSymbol]; - nextTick(() => { - resolve(rid); - }); + nextTick(() => resolve(rid)); }); }); socket[kSession] = this; @@ -882,6 +882,7 @@ export class ClientHttp2Stream extends Duplex { trailersReady: false, endAfterHeaders: false, shutdownWritableCalled: false, + serverEndedCall: false, }; this[kDenoResponse] = undefined; this[kDenoRid] = undefined; @@ -1109,7 +1110,9 @@ export class ClientHttp2Stream extends Duplex { } debugHttp2(">>> chunk", chunk, finished, this[kDenoResponse].bodyRid); - if (chunk === null) { + if (finished || chunk === null) { + this[kState].serverEndedCall = true; + const trailerList = await op_http2_client_get_response_trailers( this[kDenoResponse].bodyRid, ); @@ -1237,7 +1240,9 @@ export class ClientHttp2Stream extends Duplex { this[kSession] = undefined; session[kMaybeDestroy](); - callback(err); + if (callback) { + callback(err); + } } [kMaybeDestroy](code = constants.NGHTTP2_NO_ERROR) { @@ -1280,6 +1285,9 @@ function shutdownWritable(stream, callback, streamRid) { if (state.flags & STREAM_FLAGS_HAS_TRAILERS) { onStreamTrailers(stream); callback(); + } else if (state.serverEndedCall) { + debugHttp2(">>> stream finished"); + callback(); } else { op_http2_client_send_data(streamRid, new Uint8Array(), true) .then(() => { diff --git a/ext/node/polyfills/inspector.js b/ext/node/polyfills/inspector.js new file mode 100644 index 00000000000000..7eb15ce9177ce7 --- /dev/null +++ b/ext/node/polyfills/inspector.js @@ -0,0 +1,210 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +// Copyright Joyent and Node contributors. All rights reserved. MIT license. + +import process from "node:process"; +import { EventEmitter } from "node:events"; +import { primordials } from "ext:core/mod.js"; +import { + op_get_extras_binding_object, + op_inspector_close, + op_inspector_connect, + op_inspector_disconnect, + op_inspector_dispatch, + op_inspector_emit_protocol_event, + op_inspector_enabled, + op_inspector_open, + op_inspector_url, + op_inspector_wait, +} from "ext:core/ops"; +import { + isUint32, + validateFunction, + validateInt32, + validateObject, + validateString, +} from "ext:deno_node/internal/validators.mjs"; +import { + ERR_INSPECTOR_ALREADY_ACTIVATED, + ERR_INSPECTOR_ALREADY_CONNECTED, + ERR_INSPECTOR_CLOSED, + ERR_INSPECTOR_COMMAND, + ERR_INSPECTOR_NOT_ACTIVE, + ERR_INSPECTOR_NOT_CONNECTED, + ERR_INSPECTOR_NOT_WORKER, +} from "ext:deno_node/internal/errors.ts"; + +const { + SymbolDispose, + JSONParse, + JSONStringify, + SafeMap, +} = primordials; + +class Session extends EventEmitter { + #connection = null; + #nextId = 1; + #messageCallbacks = new SafeMap(); + + connect() { + if (this.#connection) { + throw new ERR_INSPECTOR_ALREADY_CONNECTED("The inspector session"); + } + this.#connection = op_inspector_connect(false, (m) => this.#onMessage(m)); + } + + connectToMainThread() { + if (isMainThread) { + throw new ERR_INSPECTOR_NOT_WORKER(); + } + if (this.#connection) { + throw new ERR_INSPECTOR_ALREADY_CONNECTED("The inspector session"); + } + this.#connection = op_inspector_connect(true, (m) => this.#onMessage(m)); + } + + #onMessage(message) { + const parsed = JSONParse(message); + try { + if (parsed.id) { + const callback = this.#messageCallbacks.get(parsed.id); + this.#messageCallbacks.delete(parsed.id); + if (callback) { + if (parsed.error) { + return callback( + new ERR_INSPECTOR_COMMAND( + parsed.error.code, + parsed.error.message, + ), + ); + } + + callback(null, parsed.result); + } + } else { + this.emit(parsed.method, parsed); + this.emit("inspectorNotification", parsed); + } + } catch (error) { + process.emitWarning(error); + } + } + + post(method, params, callback) { + validateString(method, "method"); + if (!callback && typeof params === "function") { + callback = params; + params = null; + } + if (params) { + validateObject(params, "params"); + } + if (callback) { + validateFunction(callback, "callback"); + } + + if (!this.#connection) { + throw new ERR_INSPECTOR_NOT_CONNECTED(); + } + const id = this.#nextId++; + const message = { id, method }; + if (params) { + message.params = params; + } + if (callback) { + this.#messageCallbacks.set(id, callback); + } + op_inspector_dispatch(this.#connection, JSONStringify(message)); + } + + disconnect() { + if (!this.#connection) { + return; + } + op_inspector_disconnect(this.#connection); + this.#connection = null; + // deno-lint-ignore prefer-primordials + for (const callback of this.#messageCallbacks.values()) { + process.nextTick(callback, new ERR_INSPECTOR_CLOSED()); + } + this.#messageCallbacks.clear(); + this.#nextId = 1; + } +} + +function open(port, host, wait) { + if (op_inspector_enabled()) { + throw new ERR_INSPECTOR_ALREADY_ACTIVATED(); + } + // inspectorOpen() currently does not typecheck its arguments and adding + // such checks would be a potentially breaking change. However, the native + // open() function requires the port to fit into a 16-bit unsigned integer, + // causing an integer overflow otherwise, so we at least need to prevent that. + if (isUint32(port)) { + validateInt32(port, "port", 0, 65535); + } else { + // equiv of handling args[0]->IsUint32() + port = undefined; + } + if (typeof host !== "string") { + // equiv of handling args[1]->IsString() + host = undefined; + } + op_inspector_open(port, host); + if (wait) { + op_inspector_wait(); + } + + return { + __proto__: null, + [SymbolDispose]() { + _debugEnd(); + }, + }; +} + +function close() { + op_inspector_close(); +} + +function url() { + return op_inspector_url(); +} + +function waitForDebugger() { + if (!op_inspector_wait()) { + throw new ERR_INSPECTOR_NOT_ACTIVE(); + } +} + +function broadcastToFrontend(eventName, params) { + validateString(eventName, "eventName"); + if (params) { + validateObject(params, "params"); + } + op_inspector_emit_protocol_event(eventName, JSONStringify(params ?? {})); +} + +const Network = { + requestWillBeSent: (params) => + broadcastToFrontend("Network.requestWillBeSent", params), + responseReceived: (params) => + broadcastToFrontend("Network.responseReceived", params), + loadingFinished: (params) => + broadcastToFrontend("Network.loadingFinished", params), + loadingFailed: (params) => + broadcastToFrontend("Network.loadingFailed", params), +}; + +const console = op_get_extras_binding_object().console; + +export { close, console, Network, open, Session, url, waitForDebugger }; + +export default { + open, + close, + url, + waitForDebugger, + console, + Session, + Network, +}; diff --git a/ext/node/polyfills/inspector.ts b/ext/node/polyfills/inspector.ts deleted file mode 100644 index 9de86ab14ffc8e..00000000000000 --- a/ext/node/polyfills/inspector.ts +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -// Copyright Joyent and Node contributors. All rights reserved. MIT license. - -import { EventEmitter } from "node:events"; -import { notImplemented } from "ext:deno_node/_utils.ts"; -import { primordials } from "ext:core/mod.js"; - -const { - SafeMap, -} = primordials; - -class Session extends EventEmitter { - #connection = null; - #nextId = 1; - #messageCallbacks = new SafeMap(); - - /** Connects the session to the inspector back-end. */ - connect() { - notImplemented("inspector.Session.prototype.connect"); - } - - /** Connects the session to the main thread - * inspector back-end. */ - connectToMainThread() { - notImplemented("inspector.Session.prototype.connectToMainThread"); - } - - /** Posts a message to the inspector back-end. */ - post( - _method: string, - _params?: Record, - _callback?: (...args: unknown[]) => void, - ) { - notImplemented("inspector.Session.prototype.post"); - } - - /** Immediately closes the session, all pending - * message callbacks will be called with an - * error. - */ - disconnect() { - notImplemented("inspector.Session.prototype.disconnect"); - } -} - -/** Activates inspector on host and port. - * See https://nodejs.org/api/inspector.html#inspectoropenport-host-wait */ -function open(_port?: number, _host?: string, _wait?: boolean) { - notImplemented("inspector.Session.prototype.open"); -} - -/** Deactivate the inspector. Blocks until there are no active connections. - * See https://nodejs.org/api/inspector.html#inspectorclose */ -function close() { - notImplemented("inspector.Session.prototype.close"); -} - -/** Return the URL of the active inspector, or undefined if there is none. - * See https://nodejs.org/api/inspector.html#inspectorurl */ -function url() { - // TODO(kt3k): returns undefined for now, which means the inspector is not activated. - return undefined; -} - -/** Blocks until a client (existing or connected later) has sent Runtime.runIfWaitingForDebugger command. - * See https://nodejs.org/api/inspector.html#inspectorwaitfordebugger */ -function waitForDebugger() { - notImplemented("inspector.wairForDebugger"); -} - -const console = globalThis.console; - -export { close, console, open, Session, url, waitForDebugger }; - -export default { - close, - console, - open, - Session, - url, - waitForDebugger, -}; diff --git a/ext/node/polyfills/inspector/promises.js b/ext/node/polyfills/inspector/promises.js new file mode 100644 index 00000000000000..3483e53f5e36f0 --- /dev/null +++ b/ext/node/polyfills/inspector/promises.js @@ -0,0 +1,20 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +// Copyright Joyent and Node contributors. All rights reserved. MIT license. + +import inspector from "node:inspector"; +import { promisify } from "ext:deno_node/internal/util.mjs"; + +class Session extends inspector.Session { + constructor() { + super(); + } +} +Session.prototype.post = promisify(inspector.Session.prototype.post); + +export * from "node:inspector"; +export { Session }; + +export default { + ...inspector, + Session, +}; diff --git a/ext/node/polyfills/internal/buffer.mjs b/ext/node/polyfills/internal/buffer.mjs index 6e43a4903166d4..dd549221fab4ad 100644 --- a/ext/node/polyfills/internal/buffer.mjs +++ b/ext/node/polyfills/internal/buffer.mjs @@ -2,11 +2,60 @@ // Copyright Joyent and Node contributors. All rights reserved. MIT license. // Copyright Feross Aboukhadijeh, and other contributors. All rights reserved. MIT license. -// TODO(petamoriken): enable prefer-primordials for node polyfills -// deno-lint-ignore-file prefer-primordials - -import { core } from "ext:core/mod.js"; -import { op_is_ascii, op_is_utf8 } from "ext:core/ops"; +import { core, primordials } from "ext:core/mod.js"; +const { + isAnyArrayBuffer, + isArrayBuffer, + isDataView, + isSharedArrayBuffer, + isTypedArray, +} = core; +const { + ArrayBufferPrototypeGetByteLength, + ArrayBufferPrototypeGetDetached, + ArrayIsArray, + ArrayPrototypeSlice, + BigInt, + DataViewPrototypeGetByteLength, + Float32Array, + Float64Array, + MathFloor, + MathMin, + Number, + NumberIsInteger, + NumberIsNaN, + NumberMAX_SAFE_INTEGER, + NumberMIN_SAFE_INTEGER, + NumberPrototypeToString, + ObjectCreate, + ObjectDefineProperty, + ObjectPrototypeIsPrototypeOf, + ObjectSetPrototypeOf, + RangeError, + SafeRegExp, + String, + StringFromCharCode, + StringPrototypeCharCodeAt, + StringPrototypeIncludes, + StringPrototypeReplace, + StringPrototypeToLowerCase, + StringPrototypeTrim, + SymbolFor, + SymbolToPrimitive, + TypeError, + TypeErrorPrototype, + TypedArrayPrototypeCopyWithin, + TypedArrayPrototypeFill, + TypedArrayPrototypeGetBuffer, + TypedArrayPrototypeGetByteLength, + TypedArrayPrototypeGetByteOffset, + TypedArrayPrototypeSet, + TypedArrayPrototypeSlice, + TypedArrayPrototypeSubarray, + Uint8Array, + Uint8ArrayPrototype, +} = primordials; +import { op_is_ascii, op_is_utf8, op_transcode } from "ext:core/ops"; import { TextDecoder, TextEncoder } from "ext:deno_web/08_text_encoding.js"; import { codes } from "ext:deno_node/internal/error_codes.ts"; @@ -24,15 +73,14 @@ import { hexToBytes, utf16leToBytes, } from "ext:deno_node/internal_binding/_utils.ts"; -import { - isAnyArrayBuffer, - isArrayBufferView, - isTypedArray, -} from "ext:deno_node/internal/util/types.ts"; import { normalizeEncoding } from "ext:deno_node/internal/util.mjs"; import { validateBuffer } from "ext:deno_node/internal/validators.mjs"; import { isUint8Array } from "ext:deno_node/internal/util/types.ts"; -import { ERR_INVALID_STATE, NodeError } from "ext:deno_node/internal/errors.ts"; +import { + ERR_INVALID_STATE, + genericNodeError, + NodeError, +} from "ext:deno_node/internal/errors.ts"; import { forgivingBase64Encode, forgivingBase64UrlEncode, @@ -46,9 +94,13 @@ const utf8Encoder = new TextEncoder(); // Temporary buffers to convert numbers. const float32Array = new Float32Array(1); -const uInt8Float32Array = new Uint8Array(float32Array.buffer); +const uInt8Float32Array = new Uint8Array( + TypedArrayPrototypeGetBuffer(float32Array), +); const float64Array = new Float64Array(1); -const uInt8Float64Array = new Uint8Array(float64Array.buffer); +const uInt8Float64Array = new Uint8Array( + TypedArrayPrototypeGetBuffer(float64Array), +); // Check endianness. float32Array[0] = -1; // 0xBF800000 @@ -60,10 +112,7 @@ export const kMaxLength = 2147483647; export const kStringMaxLength = 536870888; const MAX_UINT32 = 2 ** 32; -const customInspectSymbol = - typeof Symbol === "function" && typeof Symbol["for"] === "function" - ? Symbol["for"]("nodejs.util.inspect.custom") - : null; +const customInspectSymbol = SymbolFor("nodejs.util.inspect.custom"); export const INSPECT_MAX_BYTES = 50; @@ -72,23 +121,25 @@ export const constants = { MAX_STRING_LENGTH: kStringMaxLength, }; -Object.defineProperty(Buffer.prototype, "parent", { +ObjectDefineProperty(Buffer.prototype, "parent", { + __proto__: null, enumerable: true, get: function () { - if (!Buffer.isBuffer(this)) { + if (!BufferIsBuffer(this)) { return void 0; } - return this.buffer; + return TypedArrayPrototypeGetBuffer(this); }, }); -Object.defineProperty(Buffer.prototype, "offset", { +ObjectDefineProperty(Buffer.prototype, "offset", { + __proto__: null, enumerable: true, get: function () { - if (!Buffer.isBuffer(this)) { + if (!BufferIsBuffer(this)) { return void 0; } - return this.byteOffset; + return TypedArrayPrototypeGetByteOffset(this); }, }); @@ -99,10 +150,21 @@ function createBuffer(length) { ); } const buf = new Uint8Array(length); - Object.setPrototypeOf(buf, Buffer.prototype); + ObjectSetPrototypeOf(buf, BufferPrototype); return buf; } +/** + * @param {ArrayBufferLike} O + * @returns {boolean} + */ +function isDetachedBuffer(O) { + if (isSharedArrayBuffer(O)) { + return false; + } + return ArrayBufferPrototypeGetDetached(O); +} + export function Buffer(arg, encodingOrOffset, length) { if (typeof arg === "number") { if (typeof encodingOrOffset === "string") { @@ -129,6 +191,7 @@ function _from(value, encodingOrOffset, length) { return fromArrayBuffer(value, encodingOrOffset, length); } + // deno-lint-ignore prefer-primordials const valueOf = value.valueOf && value.valueOf(); if ( valueOf != null && @@ -143,8 +206,8 @@ function _from(value, encodingOrOffset, length) { return b; } - if (typeof value[Symbol.toPrimitive] === "function") { - const primitive = value[Symbol.toPrimitive]("string"); + if (typeof value[SymbolToPrimitive] === "function") { + const primitive = value[SymbolToPrimitive]("string"); if (typeof primitive === "string") { return fromString(primitive, encodingOrOffset); } @@ -158,13 +221,19 @@ function _from(value, encodingOrOffset, length) { ); } -Buffer.from = function from(value, encodingOrOffset, length) { +const BufferFrom = Buffer.from = function from( + value, + encodingOrOffset, + length, +) { return _from(value, encodingOrOffset, length); }; -Object.setPrototypeOf(Buffer.prototype, Uint8Array.prototype); +const BufferPrototype = Buffer.prototype; -Object.setPrototypeOf(Buffer, Uint8Array); +ObjectSetPrototypeOf(Buffer.prototype, Uint8ArrayPrototype); + +ObjectSetPrototypeOf(Buffer, Uint8Array); function assertSize(size) { validateNumber(size, "size", 0, kMaxLength); @@ -182,6 +251,7 @@ function _alloc(size, fill, encoding) { encoding, ); } + // deno-lint-ignore prefer-primordials return buffer.fill(fill, encoding); } return buffer; @@ -208,13 +278,14 @@ function fromString(string, encoding) { if (typeof encoding !== "string" || encoding === "") { encoding = "utf8"; } - if (!Buffer.isEncoding(encoding)) { + if (!BufferIsEncoding(encoding)) { throw new codes.ERR_UNKNOWN_ENCODING(encoding); } const length = byteLength(string, encoding) | 0; let buf = createBuffer(length); const actual = buf.write(string, encoding); if (actual !== length) { + // deno-lint-ignore prefer-primordials buf = buf.slice(0, actual); } return buf; @@ -222,11 +293,12 @@ function fromString(string, encoding) { function fromArrayLike(obj) { const buf = new Uint8Array(obj); - Object.setPrototypeOf(buf, Buffer.prototype); + ObjectSetPrototypeOf(buf, BufferPrototype); return buf; } function fromObject(obj) { + // deno-lint-ignore prefer-primordials if (obj.length !== undefined || isAnyArrayBuffer(obj.buffer)) { if (typeof obj.length !== "number") { return createBuffer(0); @@ -235,7 +307,7 @@ function fromObject(obj) { return fromArrayLike(obj); } - if (obj.type === "Buffer" && Array.isArray(obj.data)) { + if (obj.type === "Buffer" && ArrayIsArray(obj.data)) { return fromArrayLike(obj.data); } } @@ -244,7 +316,7 @@ function checked(length) { if (length >= kMaxLength) { throw new RangeError( "Attempt to allocate Buffer larger than maximum size: 0x" + - kMaxLength.toString(16) + " bytes", + NumberPrototypeToString(kMaxLength, 16) + " bytes", ); } return length | 0; @@ -252,25 +324,33 @@ function checked(length) { export function SlowBuffer(length) { assertSize(length); - return Buffer.alloc(+length); + return _alloc(+length); } -Object.setPrototypeOf(SlowBuffer.prototype, Uint8Array.prototype); +ObjectSetPrototypeOf(SlowBuffer.prototype, Uint8ArrayPrototype); -Object.setPrototypeOf(SlowBuffer, Uint8Array); +ObjectSetPrototypeOf(SlowBuffer, Uint8Array); -Buffer.isBuffer = function isBuffer(b) { - return b != null && b._isBuffer === true && b !== Buffer.prototype; +const BufferIsBuffer = Buffer.isBuffer = function isBuffer(b) { + return b != null && b._isBuffer === true && b !== BufferPrototype; }; -Buffer.compare = function compare(a, b) { - if (isInstance(a, Uint8Array)) { - a = Buffer.from(a, a.offset, a.byteLength); +const BufferCompare = Buffer.compare = function compare(a, b) { + if (isUint8Array(a)) { + a = BufferFrom( + a, + TypedArrayPrototypeGetByteOffset(a), + TypedArrayPrototypeGetByteLength(a), + ); } - if (isInstance(b, Uint8Array)) { - b = Buffer.from(b, b.offset, b.byteLength); + if (isUint8Array(b)) { + b = BufferFrom( + b, + TypedArrayPrototypeGetByteOffset(b), + TypedArrayPrototypeGetByteLength(b), + ); } - if (!Buffer.isBuffer(a) || !Buffer.isBuffer(b)) { + if (!BufferIsBuffer(a) || !BufferIsBuffer(b)) { throw new TypeError( 'The "buf1", "buf2" arguments must be one of type Buffer or Uint8Array', ); @@ -280,7 +360,7 @@ Buffer.compare = function compare(a, b) { } let x = a.length; let y = b.length; - for (let i = 0, len = Math.min(x, y); i < len; ++i) { + for (let i = 0, len = MathMin(x, y); i < len; ++i) { if (a[i] !== b[i]) { x = a[i]; y = b[i]; @@ -296,18 +376,18 @@ Buffer.compare = function compare(a, b) { return 0; }; -Buffer.isEncoding = function isEncoding(encoding) { +const BufferIsEncoding = Buffer.isEncoding = function isEncoding(encoding) { return typeof encoding === "string" && encoding.length !== 0 && normalizeEncoding(encoding) !== undefined; }; Buffer.concat = function concat(list, length) { - if (!Array.isArray(list)) { + if (!ArrayIsArray(list)) { throw new codes.ERR_INVALID_ARG_TYPE("list", "Array", list); } if (list.length === 0) { - return Buffer.alloc(0); + return _alloc(0); } if (length === undefined) { @@ -321,7 +401,7 @@ Buffer.concat = function concat(list, length) { validateOffset(length, "length"); } - const buffer = Buffer.allocUnsafe(length); + const buffer = _allocUnsafe(length); let pos = 0; for (let i = 0; i < list.length; i++) { const buf = list[i]; @@ -342,7 +422,7 @@ Buffer.concat = function concat(list, length) { // Zero-fill the remaining bytes if the specified `length` was more than // the actual total length, i.e. if we have some remaining allocated bytes // there were not initialized. - buffer.fill(0, pos, length); + TypedArrayPrototypeFill(buffer, 0, pos, length); } return buffer; @@ -350,7 +430,18 @@ Buffer.concat = function concat(list, length) { function byteLength(string, encoding) { if (typeof string !== "string") { - if (isArrayBufferView(string) || isAnyArrayBuffer(string)) { + if (isTypedArray(string)) { + return TypedArrayPrototypeGetByteLength(string); + } + if (isDataView(string)) { + return DataViewPrototypeGetByteLength(string); + } + if (isArrayBuffer(string)) { + return ArrayBufferPrototypeGetByteLength(string); + } + if (isSharedArrayBuffer(string)) { + // TODO(petamoriken): add SharedArayBuffer to primordials + // deno-lint-ignore prefer-primordials return string.byteLength; } @@ -459,6 +550,7 @@ Buffer.prototype.toString = function toString(encoding, start, end) { throw new codes.ERR_UNKNOWN_ENCODING(encoding); } + // deno-lint-ignore prefer-primordials return ops.slice(this, start, end); }; @@ -475,22 +567,29 @@ Buffer.prototype.equals = function equals(b) { if (this === b) { return true; } - return Buffer.compare(this, b) === 0; + return BufferCompare(this, b) === 0; }; -Buffer.prototype.inspect = function inspect() { - let str = ""; - const max = INSPECT_MAX_BYTES; - str = this.toString("hex", 0, max).replace(/(.{2})/g, "$1 ").trim(); - if (this.length > max) { - str += " ... "; - } - return ""; -}; +const SPACER_PATTERN = new SafeRegExp(/(.{2})/g); -if (customInspectSymbol) { - Buffer.prototype[customInspectSymbol] = Buffer.prototype.inspect; -} +Buffer.prototype[customInspectSymbol] = + Buffer.prototype.inspect = + function inspect() { + let str = ""; + const max = INSPECT_MAX_BYTES; + str = StringPrototypeTrim( + StringPrototypeReplace( + // deno-lint-ignore prefer-primordials + this.toString("hex", 0, max), + SPACER_PATTERN, + "$1 ", + ), + ); + if (this.length > max) { + str += " ... "; + } + return ""; + }; Buffer.prototype.compare = function compare( target, @@ -499,10 +598,14 @@ Buffer.prototype.compare = function compare( thisStart, thisEnd, ) { - if (isInstance(target, Uint8Array)) { - target = Buffer.from(target, target.offset, target.byteLength); + if (isUint8Array(target)) { + target = BufferFrom( + target, + TypedArrayPrototypeGetByteOffset(target), + TypedArrayPrototypeGetByteLength(target), + ); } - if (!Buffer.isBuffer(target)) { + if (!BufferIsBuffer(target)) { throw new codes.ERR_INVALID_ARG_TYPE( "target", ["Buffer", "Uint8Array"], @@ -559,8 +662,9 @@ Buffer.prototype.compare = function compare( } let x = thisEnd - thisStart; let y = end - start; - const len = Math.min(x, y); - const thisCopy = this.slice(thisStart, thisEnd); + const len = MathMin(x, y); + const thisCopy = TypedArrayPrototypeSlice(this, thisStart, thisEnd); + // deno-lint-ignore prefer-primordials const targetCopy = target.slice(start, end); for (let i = 0; i < len; ++i) { if (thisCopy[i] !== targetCopy[i]) { @@ -590,7 +694,8 @@ function bidirectionalIndexOf(buffer, val, byteOffset, encoding, dir) { byteOffset = -0x80000000; } byteOffset = +byteOffset; - if (Number.isNaN(byteOffset)) { + if (NumberIsNaN(byteOffset)) { + // deno-lint-ignore prefer-primordials byteOffset = dir ? 0 : (buffer.length || buffer.byteLength); } dir = !!dir; @@ -610,6 +715,7 @@ function bidirectionalIndexOf(buffer, val, byteOffset, encoding, dir) { if (ops === undefined) { throw new codes.ERR_UNKNOWN_ENCODING(encoding); } + // deno-lint-ignore prefer-primordials return ops.indexOf(buffer, val, byteOffset, dir); } @@ -626,6 +732,7 @@ function bidirectionalIndexOf(buffer, val, byteOffset, encoding, dir) { } Buffer.prototype.includes = function includes(val, byteOffset, encoding) { + // deno-lint-ignore prefer-primordials return this.indexOf(val, byteOffset, encoding) !== -1; }; @@ -645,7 +752,7 @@ Buffer.prototype.asciiSlice = function asciiSlice(offset, length) { if (offset === 0 && length === this.length) { return bytesToAscii(this); } else { - return bytesToAscii(this.slice(offset, length)); + return bytesToAscii(TypedArrayPrototypeSlice(this, offset, length)); } }; @@ -660,7 +767,9 @@ Buffer.prototype.base64Slice = function base64Slice( if (offset === 0 && length === this.length) { return forgivingBase64Encode(this); } else { - return forgivingBase64Encode(this.slice(offset, length)); + return forgivingBase64Encode( + TypedArrayPrototypeSlice(this, offset, length), + ); } }; @@ -679,7 +788,9 @@ Buffer.prototype.base64urlSlice = function base64urlSlice( if (offset === 0 && length === this.length) { return forgivingBase64UrlEncode(this); } else { - return forgivingBase64UrlEncode(this.slice(offset, length)); + return forgivingBase64UrlEncode( + TypedArrayPrototypeSlice(this, offset, length), + ); } }; @@ -724,7 +835,7 @@ Buffer.prototype.ucs2Slice = function ucs2Slice(offset, length) { if (offset === 0 && length === this.length) { return bytesToUtf16le(this); } else { - return bytesToUtf16le(this.slice(offset, length)); + return bytesToUtf16le(TypedArrayPrototypeSlice(this, offset, length)); } }; @@ -743,9 +854,9 @@ Buffer.prototype.utf8Slice = function utf8Slice(string, offset, length) { Buffer.prototype.utf8Write = function utf8Write(string, offset, length) { offset = offset || 0; - const maxLength = Math.min(length || Infinity, this.length - offset); + const maxLength = MathMin(length || Infinity, this.length - offset); const buf = offset || maxLength < this.length - ? this.subarray(offset, maxLength + offset) + ? TypedArrayPrototypeSubarray(this, offset, maxLength + offset) : this; return utf8Encoder.encodeInto(string, buf).written; }; @@ -797,7 +908,7 @@ Buffer.prototype.write = function write(string, offset, length, encoding) { Buffer.prototype.toJSON = function toJSON() { return { type: "Buffer", - data: Array.prototype.slice.call(this._arr || this, 0), + data: ArrayPrototypeSlice(this._arr || this, 0), }; }; function fromArrayBuffer(obj, byteOffset, length) { @@ -806,11 +917,12 @@ function fromArrayBuffer(obj, byteOffset, length) { byteOffset = 0; } else { byteOffset = +byteOffset; - if (Number.isNaN(byteOffset)) { + if (NumberIsNaN(byteOffset)) { byteOffset = 0; } } + // deno-lint-ignore prefer-primordials const maxLength = obj.byteLength - byteOffset; if (maxLength < 0) { @@ -832,7 +944,7 @@ function fromArrayBuffer(obj, byteOffset, length) { } const buffer = new Uint8Array(obj, byteOffset, length); - Object.setPrototypeOf(buffer, Buffer.prototype); + ObjectSetPrototypeOf(buffer, BufferPrototype); return buffer; } @@ -840,6 +952,7 @@ function _base64Slice(buf, start, end) { if (start === 0 && end === buf.length) { return forgivingBase64Encode(buf); } else { + // deno-lint-ignore prefer-primordials return forgivingBase64Encode(buf.slice(start, end)); } } @@ -848,9 +961,10 @@ const decoder = new TextDecoder(); function _utf8Slice(buf, start, end) { try { + // deno-lint-ignore prefer-primordials return decoder.decode(buf.slice(start, end)); } catch (err) { - if (err instanceof TypeError) { + if (ObjectPrototypeIsPrototypeOf(TypeErrorPrototype, err)) { throw new NodeError("ERR_STRING_TOO_LONG", "String too long"); } throw err; @@ -859,9 +973,9 @@ function _utf8Slice(buf, start, end) { function _latin1Slice(buf, start, end) { let ret = ""; - end = Math.min(buf.length, end); + end = MathMin(buf.length, end); for (let i = start; i < end; ++i) { - ret += String.fromCharCode(buf[i]); + ret += StringFromCharCode(buf[i]); } return ret; } @@ -990,42 +1104,38 @@ Buffer.prototype.readUint32BE = Buffer.prototype.readUInt32BE = readUInt32BE; Buffer.prototype.readBigUint64LE = Buffer.prototype.readBigUInt64LE = - defineBigIntMethod( - function readBigUInt64LE(offset) { - offset = offset >>> 0; - validateNumber(offset, "offset"); - const first = this[offset]; - const last = this[offset + 7]; - if (first === void 0 || last === void 0) { - boundsError(offset, this.length - 8); - } - const lo = first + this[++offset] * 2 ** 8 + - this[++offset] * 2 ** 16 + - this[++offset] * 2 ** 24; - const hi = this[++offset] + this[++offset] * 2 ** 8 + - this[++offset] * 2 ** 16 + last * 2 ** 24; - return BigInt(lo) + (BigInt(hi) << BigInt(32)); - }, - ); + function readBigUInt64LE(offset) { + offset = offset >>> 0; + validateNumber(offset, "offset"); + const first = this[offset]; + const last = this[offset + 7]; + if (first === void 0 || last === void 0) { + boundsError(offset, this.length - 8); + } + const lo = first + this[++offset] * 2 ** 8 + + this[++offset] * 2 ** 16 + + this[++offset] * 2 ** 24; + const hi = this[++offset] + this[++offset] * 2 ** 8 + + this[++offset] * 2 ** 16 + last * 2 ** 24; + return BigInt(lo) + (BigInt(hi) << 32n); + }; Buffer.prototype.readBigUint64BE = Buffer.prototype.readBigUInt64BE = - defineBigIntMethod( - function readBigUInt64BE(offset) { - offset = offset >>> 0; - validateNumber(offset, "offset"); - const first = this[offset]; - const last = this[offset + 7]; - if (first === void 0 || last === void 0) { - boundsError(offset, this.length - 8); - } - const hi = first * 2 ** 24 + this[++offset] * 2 ** 16 + - this[++offset] * 2 ** 8 + this[++offset]; - const lo = this[++offset] * 2 ** 24 + this[++offset] * 2 ** 16 + - this[++offset] * 2 ** 8 + last; - return (BigInt(hi) << BigInt(32)) + BigInt(lo); - }, - ); + function readBigUInt64BE(offset) { + offset = offset >>> 0; + validateNumber(offset, "offset"); + const first = this[offset]; + const last = this[offset + 7]; + if (first === void 0 || last === void 0) { + boundsError(offset, this.length - 8); + } + const hi = first * 2 ** 24 + this[++offset] * 2 ** 16 + + this[++offset] * 2 ** 8 + this[++offset]; + const lo = this[++offset] * 2 ** 24 + this[++offset] * 2 ** 16 + + this[++offset] * 2 ** 8 + last; + return (BigInt(hi) << 32n) + BigInt(lo); + }; Buffer.prototype.readIntLE = function readIntLE( offset, @@ -1144,43 +1254,39 @@ Buffer.prototype.readInt32BE = function readInt32BE(offset = 0) { last; }; -Buffer.prototype.readBigInt64LE = defineBigIntMethod( - function readBigInt64LE(offset) { - offset = offset >>> 0; - validateNumber(offset, "offset"); - const first = this[offset]; - const last = this[offset + 7]; - if (first === void 0 || last === void 0) { - boundsError(offset, this.length - 8); - } - const val = this[offset + 4] + this[offset + 5] * 2 ** 8 + - this[offset + 6] * 2 ** 16 + (last << 24); - return (BigInt(val) << BigInt(32)) + - BigInt( - first + this[++offset] * 2 ** 8 + this[++offset] * 2 ** 16 + - this[++offset] * 2 ** 24, - ); - }, -); +Buffer.prototype.readBigInt64LE = function readBigInt64LE(offset) { + offset = offset >>> 0; + validateNumber(offset, "offset"); + const first = this[offset]; + const last = this[offset + 7]; + if (first === void 0 || last === void 0) { + boundsError(offset, this.length - 8); + } + const val = this[offset + 4] + this[offset + 5] * 2 ** 8 + + this[offset + 6] * 2 ** 16 + (last << 24); + return (BigInt(val) << 32n) + + BigInt( + first + this[++offset] * 2 ** 8 + this[++offset] * 2 ** 16 + + this[++offset] * 2 ** 24, + ); +}; -Buffer.prototype.readBigInt64BE = defineBigIntMethod( - function readBigInt64BE(offset) { - offset = offset >>> 0; - validateNumber(offset, "offset"); - const first = this[offset]; - const last = this[offset + 7]; - if (first === void 0 || last === void 0) { - boundsError(offset, this.length - 8); - } - const val = (first << 24) + this[++offset] * 2 ** 16 + - this[++offset] * 2 ** 8 + this[++offset]; - return (BigInt(val) << BigInt(32)) + - BigInt( - this[++offset] * 2 ** 24 + this[++offset] * 2 ** 16 + - this[++offset] * 2 ** 8 + last, - ); - }, -); +Buffer.prototype.readBigInt64BE = function readBigInt64BE(offset) { + offset = offset >>> 0; + validateNumber(offset, "offset"); + const first = this[offset]; + const last = this[offset + 7]; + if (first === void 0 || last === void 0) { + boundsError(offset, this.length - 8); + } + const val = (first << 24) + this[++offset] * 2 ** 16 + + this[++offset] * 2 ** 8 + this[++offset]; + return (BigInt(val) << 32n) + + BigInt( + this[++offset] * 2 ** 24 + this[++offset] * 2 ** 16 + + this[++offset] * 2 ** 8 + last, + ); +}; Buffer.prototype.readFloatLE = function readFloatLE(offset) { return bigEndian @@ -1289,7 +1395,7 @@ Buffer.prototype.writeUint32BE = function wrtBigUInt64LE(buf, value, offset, min, max) { checkIntBI(value, min, max, buf, offset, 7); - let lo = Number(value & BigInt(4294967295)); + let lo = Number(value & 4294967295n); buf[offset++] = lo; lo = lo >> 8; buf[offset++] = lo; @@ -1297,7 +1403,7 @@ function wrtBigUInt64LE(buf, value, offset, min, max) { buf[offset++] = lo; lo = lo >> 8; buf[offset++] = lo; - let hi = Number(value >> BigInt(32) & BigInt(4294967295)); + let hi = Number(value >> 32n & 4294967295n); buf[offset++] = hi; hi = hi >> 8; buf[offset++] = hi; @@ -1310,7 +1416,7 @@ function wrtBigUInt64LE(buf, value, offset, min, max) { function wrtBigUInt64BE(buf, value, offset, min, max) { checkIntBI(value, min, max, buf, offset, 7); - let lo = Number(value & BigInt(4294967295)); + let lo = Number(value & 4294967295n); buf[offset + 7] = lo; lo = lo >> 8; buf[offset + 6] = lo; @@ -1318,7 +1424,7 @@ function wrtBigUInt64BE(buf, value, offset, min, max) { buf[offset + 5] = lo; lo = lo >> 8; buf[offset + 4] = lo; - let hi = Number(value >> BigInt(32) & BigInt(4294967295)); + let hi = Number(value >> 32n & 4294967295n); buf[offset + 3] = hi; hi = hi >> 8; buf[offset + 2] = hi; @@ -1331,31 +1437,27 @@ function wrtBigUInt64BE(buf, value, offset, min, max) { Buffer.prototype.writeBigUint64LE = Buffer.prototype.writeBigUInt64LE = - defineBigIntMethod( - function writeBigUInt64LE(value, offset = 0) { - return wrtBigUInt64LE( - this, - value, - offset, - BigInt(0), - BigInt("0xffffffffffffffff"), - ); - }, - ); + function writeBigUInt64LE(value, offset = 0) { + return wrtBigUInt64LE( + this, + value, + offset, + 0n, + 0xffffffffffffffffn, + ); + }; Buffer.prototype.writeBigUint64BE = Buffer.prototype.writeBigUInt64BE = - defineBigIntMethod( - function writeBigUInt64BE(value, offset = 0) { - return wrtBigUInt64BE( - this, - value, - offset, - BigInt(0), - BigInt("0xffffffffffffffff"), - ); - }, - ); + function writeBigUInt64BE(value, offset = 0) { + return wrtBigUInt64BE( + this, + value, + offset, + 0n, + 0xffffffffffffffffn, + ); + }; Buffer.prototype.writeIntLE = function writeIntLE( value, @@ -1446,29 +1548,25 @@ Buffer.prototype.writeInt32BE = function writeInt32BE(value, offset = 0) { return writeU_Int32BE(this, value, offset, -0x80000000, 0x7fffffff); }; -Buffer.prototype.writeBigInt64LE = defineBigIntMethod( - function writeBigInt64LE(value, offset = 0) { - return wrtBigUInt64LE( - this, - value, - offset, - -BigInt("0x8000000000000000"), - BigInt("0x7fffffffffffffff"), - ); - }, -); +Buffer.prototype.writeBigInt64LE = function writeBigInt64LE(value, offset = 0) { + return wrtBigUInt64LE( + this, + value, + offset, + -0x8000000000000000n, + 0x7fffffffffffffffn, + ); +}; -Buffer.prototype.writeBigInt64BE = defineBigIntMethod( - function writeBigInt64BE(value, offset = 0) { - return wrtBigUInt64BE( - this, - value, - offset, - -BigInt("0x8000000000000000"), - BigInt("0x7fffffffffffffff"), - ); - }, -); +Buffer.prototype.writeBigInt64BE = function writeBigInt64BE(value, offset = 0) { + return wrtBigUInt64BE( + this, + value, + offset, + -0x8000000000000000n, + 0x7fffffffffffffffn, + ); +}; Buffer.prototype.writeFloatLE = function writeFloatLE( value, @@ -1596,14 +1694,12 @@ Buffer.prototype.copy = function copy( } const len = sourceEnd - sourceStart; - if ( - this === target && typeof Uint8Array.prototype.copyWithin === "function" - ) { - this.copyWithin(targetStart, sourceStart, sourceEnd); + if (this === target) { + TypedArrayPrototypeCopyWithin(this, targetStart, sourceStart, sourceEnd); } else { - Uint8Array.prototype.set.call( + TypedArrayPrototypeSet( target, - this.subarray(sourceStart, sourceEnd), + TypedArrayPrototypeSubarray(this, sourceStart, sourceEnd), targetStart, ); } @@ -1623,11 +1719,11 @@ Buffer.prototype.fill = function fill(val, start, end, encoding) { if (encoding !== void 0 && typeof encoding !== "string") { throw new TypeError("encoding must be a string"); } - if (typeof encoding === "string" && !Buffer.isEncoding(encoding)) { + if (typeof encoding === "string" && !BufferIsEncoding(encoding)) { throw new TypeError("Unknown encoding: " + encoding); } if (val.length === 1) { - const code = val.charCodeAt(0); + const code = StringPrototypeCharCodeAt(val, 0); if (encoding === "utf8" && code < 128 || encoding === "latin1") { val = code; } @@ -1654,7 +1750,7 @@ Buffer.prototype.fill = function fill(val, start, end, encoding) { this[i] = val; } } else { - const bytes = Buffer.isBuffer(val) ? val : Buffer.from(val, encoding); + const bytes = BufferIsBuffer(val) ? val : BufferFrom(val, encoding); const len = bytes.length; if (len === 0) { throw new codes.ERR_INVALID_ARG_VALUE( @@ -1681,7 +1777,7 @@ function checkIntBI(value, min, max, buf, offset, byteLength2) { const n = typeof min === "bigint" ? "n" : ""; let range; if (byteLength2 > 3) { - if (min === 0 || min === BigInt(0)) { + if (min === 0 || min === 0n) { range = `>= 0${n} and < 2${n} ** ${(byteLength2 + 1) * 8}${n}`; } else { range = `>= -(2${n} ** ${(byteLength2 + 1) * 8 - 1}${n}) and < 2 ** ${ @@ -1706,7 +1802,7 @@ function checkIntBI(value, min, max, buf, offset, byteLength2) { function blitBuffer(src, dst, offset, byteLength = Infinity) { const srcLength = src.length; // Establish the number of bytes to be written - const bytesToWrite = Math.min( + const bytesToWrite = MathMin( // If byte length is defined in the call, then it sets an upper bound, // otherwise it is Infinity and is never chosen. byteLength, @@ -1726,15 +1822,9 @@ function blitBuffer(src, dst, offset, byteLength = Infinity) { return bytesToWrite; } -function isInstance(obj, type) { - return obj instanceof type || - obj != null && obj.constructor != null && - obj.constructor.name != null && obj.constructor.name === type.name; -} - const hexSliceLookupTable = function () { const alphabet = "0123456789abcdef"; - const table = new Array(256); + const table = []; for (let i = 0; i < 16; ++i) { const i16 = i * 16; for (let j = 0; j < 16; ++j) { @@ -1744,14 +1834,6 @@ const hexSliceLookupTable = function () { return table; }(); -function defineBigIntMethod(fn) { - return typeof BigInt === "undefined" ? BufferBigIntNotDefined : fn; -} - -function BufferBigIntNotDefined() { - throw new Error("BigInt not supported"); -} - export function readUInt48LE(buf, offset = 0) { validateNumber(offset, "offset"); const first = buf[offset]; @@ -2075,10 +2157,10 @@ export function byteLengthUtf8(str) { function base64ByteLength(str, bytes) { // Handle padding - if (str.charCodeAt(bytes - 1) === 0x3D) { + if (StringPrototypeCharCodeAt(str, bytes - 1) === 0x3D) { bytes--; } - if (bytes > 1 && str.charCodeAt(bytes - 1) === 0x3D) { + if (bytes > 1 && StringPrototypeCharCodeAt(str, bytes - 1) === 0x3D) { bytes--; } @@ -2086,7 +2168,7 @@ function base64ByteLength(str, bytes) { return (bytes * 3) >>> 2; } -export const encodingsMap = Object.create(null); +export const encodingsMap = ObjectCreate(null); for (let i = 0; i < encodings.length; ++i) { encodingsMap[encodings[i]] = i; } @@ -2216,7 +2298,7 @@ export const encodingOps = { }; export function getEncodingOps(encoding) { - encoding = String(encoding).toLowerCase(); + encoding = StringPrototypeToLowerCase(String(encoding)); switch (encoding.length) { case 4: if (encoding === "utf8") return encodingOps.utf8; @@ -2256,6 +2338,14 @@ export function getEncodingOps(encoding) { } } +/** + * @param {Buffer} source + * @param {Buffer} target + * @param {number} targetStart + * @param {number} sourceStart + * @param {number} sourceEnd + * @returns {number} + */ export function _copyActual( source, target, @@ -2274,6 +2364,7 @@ export function _copyActual( } if (sourceStart !== 0 || sourceEnd < source.length) { + // deno-lint-ignore prefer-primordials source = new Uint8Array(source.buffer, source.byteOffset + sourceStart, nb); } @@ -2283,7 +2374,7 @@ export function _copyActual( } export function boundsError(value, length, type) { - if (Math.floor(value) !== value) { + if (MathFloor(value) !== value) { validateNumber(value, type); throw new codes.ERR_OUT_OF_RANGE(type || "offset", "an integer", value); } @@ -2306,7 +2397,7 @@ export function validateNumber(value, name, min = undefined, max) { if ( (min != null && value < min) || (max != null && value > max) || - ((min != null || max != null) && Number.isNaN(value)) + ((min != null || max != null) && NumberIsNaN(value)) ) { throw new codes.ERR_OUT_OF_RANGE( name, @@ -2340,11 +2431,11 @@ function checkInt(value, min, max, buf, offset, byteLength) { export function toInteger(n, defaultVal) { n = +n; if ( - !Number.isNaN(n) && - n >= Number.MIN_SAFE_INTEGER && - n <= Number.MAX_SAFE_INTEGER + !NumberIsNaN(n) && + n >= NumberMIN_SAFE_INTEGER && + n <= NumberMAX_SAFE_INTEGER ) { - return ((n % 1) === 0 ? n : Math.floor(n)); + return ((n % 1) === 0 ? n : MathFloor(n)); } return defaultVal; } @@ -2417,7 +2508,7 @@ export function writeU_Int48BE(buf, value, offset, min, max) { value = +value; checkInt(value, min, max, buf, offset, 5); - const newVal = Math.floor(value * 2 ** -32); + const newVal = MathFloor(value * 2 ** -32); buf[offset++] = newVal >>> 8; buf[offset++] = newVal; buf[offset + 3] = value; @@ -2435,7 +2526,7 @@ export function writeU_Int40BE(buf, value, offset, min, max) { value = +value; checkInt(value, min, max, buf, offset, 4); - buf[offset++] = Math.floor(value * 2 ** -32); + buf[offset++] = MathFloor(value * 2 ** -32); buf[offset + 3] = value; value = value >>> 8; buf[offset + 2] = value; @@ -2478,12 +2569,12 @@ export function validateOffset( value, name, min = 0, - max = Number.MAX_SAFE_INTEGER, + max = NumberMAX_SAFE_INTEGER, ) { if (typeof value !== "number") { throw new codes.ERR_INVALID_ARG_TYPE(name, "number", value); } - if (!Number.isInteger(value)) { + if (!NumberIsInteger(value)) { throw new codes.ERR_OUT_OF_RANGE(name, "an integer", value); } if (value < min || value > max) { @@ -2496,7 +2587,7 @@ export function writeU_Int48LE(buf, value, offset, min, max) { value = +value; checkInt(value, min, max, buf, offset, 5); - const newVal = Math.floor(value * 2 ** -32); + const newVal = MathFloor(value * 2 ** -32); buf[offset++] = value; value = value >>> 8; buf[offset++] = value; @@ -2522,7 +2613,7 @@ export function writeU_Int40LE(buf, value, offset, min, max) { buf[offset++] = value; value = value >>> 8; buf[offset++] = value; - buf[offset++] = Math.floor(newVal * 2 ** -32); + buf[offset++] = MathFloor(newVal * 2 ** -32); return offset; } @@ -2556,14 +2647,14 @@ export function writeU_Int24LE(buf, value, offset, min, max) { export function isUtf8(input) { if (isTypedArray(input)) { - if (input.buffer.detached) { + if (isDetachedBuffer(TypedArrayPrototypeGetBuffer(input))) { throw new ERR_INVALID_STATE("Cannot validate on a detached buffer"); } return op_is_utf8(input); } if (isAnyArrayBuffer(input)) { - if (input.detached) { + if (isDetachedBuffer(input)) { throw new ERR_INVALID_STATE("Cannot validate on a detached buffer"); } return op_is_utf8(new Uint8Array(input)); @@ -2578,14 +2669,14 @@ export function isUtf8(input) { export function isAscii(input) { if (isTypedArray(input)) { - if (input.buffer.detached) { + if (isDetachedBuffer(TypedArrayPrototypeGetBuffer(input))) { throw new ERR_INVALID_STATE("Cannot validate on a detached buffer"); } return op_is_ascii(input); } if (isAnyArrayBuffer(input)) { - if (input.detached) { + if (isDetachedBuffer(input)) { throw new ERR_INVALID_STATE("Cannot validate on a detached buffer"); } return op_is_ascii(new Uint8Array(input)); @@ -2598,6 +2689,48 @@ export function isAscii(input) { ], input); } +export function transcode(source, fromEnco, toEnco) { + if (!isUint8Array(source)) { + throw new codes.ERR_INVALID_ARG_TYPE( + "source", + ["Buffer", "Uint8Array"], + source, + ); + } + if (source.length === 0) { + return Buffer.alloc(0); + } + const code = "U_ILLEGAL_ARGUMENT_ERROR"; + const illegalArgumentError = genericNodeError( + `Unable to transcode Buffer [${code}]`, + { code: code, errno: 1 }, + ); + fromEnco = normalizeEncoding(fromEnco); + toEnco = normalizeEncoding(toEnco); + if (!fromEnco || !toEnco) { + throw illegalArgumentError; + } + // Return the provided source when transcode is not required + // for the from/to encoding pair. + const returnSource = fromEnco === toEnco || + fromEnco === "ascii" && toEnco === "utf8" || + fromEnco === "ascii" && toEnco === "latin1"; + if (returnSource) { + return Buffer.from(source); + } + + try { + const result = op_transcode(new Uint8Array(source), fromEnco, toEnco); + return Buffer.from(result, toEnco); + } catch (err) { + if (StringPrototypeIncludes(err.message, "Unable to transcode Buffer")) { + throw illegalArgumentError; + } else { + throw err; + } + } +} + export default { atob, btoa, @@ -2610,4 +2743,5 @@ export default { kMaxLength, kStringMaxLength, SlowBuffer, + transcode, }; diff --git a/ext/node/polyfills/internal/child_process.ts b/ext/node/polyfills/internal/child_process.ts index 56fc21f354cc93..cfff1079ff5986 100644 --- a/ext/node/polyfills/internal/child_process.ts +++ b/ext/node/polyfills/internal/child_process.ts @@ -56,7 +56,12 @@ import { StringPrototypeSlice } from "ext:deno_node/internal/primordials.mjs"; import { StreamBase } from "ext:deno_node/internal_binding/stream_wrap.ts"; import { Pipe, socketType } from "ext:deno_node/internal_binding/pipe_wrap.ts"; import { Socket } from "node:net"; -import { kDetached, kExtraStdio, kIpc } from "ext:runtime/40_process.js"; +import { + kDetached, + kExtraStdio, + kIpc, + kNeedsNpmProcessState, +} from "ext:runtime/40_process.js"; export function mapValues( record: Readonly>, @@ -281,6 +286,8 @@ export class ChildProcess extends EventEmitter { [kIpc]: ipc, // internal [kExtraStdio]: extraStdioNormalized, [kDetached]: detached, + // deno-lint-ignore no-explicit-any + [kNeedsNpmProcessState]: (options ?? {} as any)[kNeedsNpmProcessState], }).spawn(); this.pid = this.#process.pid; @@ -1184,8 +1191,12 @@ function toDenoArgs(args: string[]): string[] { } if (flagInfo === undefined) { - // Not a known flag that expects a value. Just copy it to the output. - denoArgs.push(arg); + if (arg === "--no-warnings") { + denoArgs.push("--quiet"); + } else { + // Not a known flag that expects a value. Just copy it to the output. + denoArgs.push(arg); + } continue; } @@ -1328,7 +1339,7 @@ export function setupChannel(target: any, ipc: number) { } } - process.nextTick(handleMessage, msg); + nextTick(handleMessage, msg); } } catch (err) { if ( @@ -1389,7 +1400,7 @@ export function setupChannel(target: any, ipc: number) { if (!target.connected) { const err = new ERR_IPC_CHANNEL_CLOSED(); if (typeof callback === "function") { - process.nextTick(callback, err); + nextTick(callback, err); } else { nextTick(() => target.emit("error", err)); } @@ -1405,7 +1416,18 @@ export function setupChannel(target: any, ipc: number) { .then(() => { control.unrefCounted(); if (callback) { - process.nextTick(callback, null); + nextTick(callback, null); + } + }, (err: Error) => { + control.unrefCounted(); + if (err instanceof Deno.errors.Interrupted) { + // Channel closed on us mid-write. + } else { + if (typeof callback === "function") { + nextTick(callback, err); + } else { + nextTick(() => target.emit("error", err)); + } } }); return queueOk[0]; @@ -1422,7 +1444,7 @@ export function setupChannel(target: any, ipc: number) { target.connected = false; target[kCanDisconnect] = false; control[kControlDisconnect](); - process.nextTick(() => { + nextTick(() => { target.channel = null; core.close(ipc); target.emit("disconnect"); diff --git a/ext/node/polyfills/internal/crypto/_randomInt.ts b/ext/node/polyfills/internal/crypto/_randomInt.ts index 7f4d703ad46132..e08b3e96399070 100644 --- a/ext/node/polyfills/internal/crypto/_randomInt.ts +++ b/ext/node/polyfills/internal/crypto/_randomInt.ts @@ -1,9 +1,15 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -// TODO(petamoriken): enable prefer-primordials for node polyfills -// deno-lint-ignore-file prefer-primordials - import { op_node_random_int } from "ext:core/ops"; +import { primordials } from "ext:core/mod.js"; +const { + Error, + MathCeil, + MathFloor, + MathPow, + NumberIsSafeInteger, + RangeError, +} = primordials; export default function randomInt(max: number): number; export default function randomInt(min: number, max: number): number; @@ -23,7 +29,9 @@ export default function randomInt( cb?: (err: Error | null, n?: number) => void, ): number | void { if (typeof max === "number" && typeof min === "number") { - [max, min] = [min, max]; + const temp = max; + max = min; + min = temp; } if (min === undefined) min = 0; else if (typeof min === "function") { @@ -32,13 +40,13 @@ export default function randomInt( } if ( - !Number.isSafeInteger(min) || - typeof max === "number" && !Number.isSafeInteger(max) + !NumberIsSafeInteger(min) || + typeof max === "number" && !NumberIsSafeInteger(max) ) { throw new Error("max or min is not a Safe Number"); } - if (max - min > Math.pow(2, 48)) { + if (max - min > MathPow(2, 48)) { throw new RangeError("max - min should be less than 2^48!"); } @@ -46,8 +54,8 @@ export default function randomInt( throw new Error("Min is bigger than Max!"); } - min = Math.ceil(min); - max = Math.floor(max); + min = MathCeil(min); + max = MathFloor(max); const result = op_node_random_int(min, max); if (cb) { diff --git a/ext/node/polyfills/internal/crypto/keygen.ts b/ext/node/polyfills/internal/crypto/keygen.ts index a40c76c0d73daa..b023ab1060c010 100644 --- a/ext/node/polyfills/internal/crypto/keygen.ts +++ b/ext/node/polyfills/internal/crypto/keygen.ts @@ -29,6 +29,8 @@ import { } from "ext:deno_node/internal/validators.mjs"; import { Buffer } from "node:buffer"; import { KeyFormat, KeyType } from "ext:deno_node/internal/crypto/types.ts"; +import process from "node:process"; +import { promisify } from "node:util"; import { op_node_generate_dh_group_key, @@ -569,7 +571,15 @@ export function generateKeyPair( privateKey: any, ) => void, ) { - createJob(kAsync, type, options).then((pair) => { + _generateKeyPair(type, options) + .then( + (res) => callback(null, res.publicKey, res.privateKey), + (err) => callback(err, null, null), + ); +} + +function _generateKeyPair(type: string, options: unknown) { + return createJob(kAsync, type, options).then((pair) => { const privateKeyHandle = op_node_get_private_key_from_pair(pair); const publicKeyHandle = op_node_get_public_key_from_pair(pair); @@ -588,12 +598,15 @@ export function generateKeyPair( } } - callback(null, publicKey, privateKey); - }).catch((err) => { - callback(err, null, null); + return { publicKey, privateKey }; }); } +Object.defineProperty(generateKeyPair, promisify.custom, { + enumerable: false, + value: _generateKeyPair, +}); + export interface KeyPairKeyObjectResult { publicKey: KeyObject; privateKey: KeyObject; diff --git a/ext/node/polyfills/internal/crypto/random.ts b/ext/node/polyfills/internal/crypto/random.ts index 4219414dc1662e..a41b8681905f03 100644 --- a/ext/node/polyfills/internal/crypto/random.ts +++ b/ext/node/polyfills/internal/crypto/random.ts @@ -38,6 +38,7 @@ import { ERR_INVALID_ARG_TYPE, ERR_OUT_OF_RANGE, } from "ext:deno_node/internal/errors.ts"; +import { Buffer } from "node:buffer"; export { default as randomBytes } from "ext:deno_node/internal/crypto/_randomBytes.ts"; export { diff --git a/ext/node/polyfills/internal/errors.ts b/ext/node/polyfills/internal/errors.ts index 51bd7a0250ef6f..61b53fa968f1fc 100644 --- a/ext/node/polyfills/internal/errors.ts +++ b/ext/node/polyfills/internal/errors.ts @@ -18,7 +18,7 @@ */ import { primordials } from "ext:core/mod.js"; -const { JSONStringify, SymbolFor } = primordials; +const { JSONStringify, SafeArrayIterator, SymbolFor } = primordials; import { format, inspect } from "ext:deno_node/internal/util/inspect.mjs"; import { codes } from "ext:deno_node/internal/error_codes.ts"; import { @@ -1874,6 +1874,11 @@ export class ERR_SOCKET_CLOSED extends NodeError { super("ERR_SOCKET_CLOSED", `Socket is closed`); } } +export class ERR_SOCKET_CONNECTION_TIMEOUT extends NodeError { + constructor() { + super("ERR_SOCKET_CONNECTION_TIMEOUT", `Socket connection timeout`); + } +} export class ERR_SOCKET_DGRAM_IS_CONNECTED extends NodeError { constructor() { super("ERR_SOCKET_DGRAM_IS_CONNECTED", `Already connected`); @@ -2385,6 +2390,15 @@ export class ERR_INVALID_RETURN_VALUE extends NodeTypeError { } } +export class ERR_NOT_IMPLEMENTED extends NodeError { + constructor(message?: string) { + super( + "ERR_NOT_IMPLEMENTED", + message ? `Not implemented: ${message}` : "Not implemented", + ); + } +} + export class ERR_INVALID_URL extends NodeTypeError { input: string; constructor(input: string) { @@ -2558,19 +2572,6 @@ export class ERR_FS_RMDIR_ENOTDIR extends NodeSystemError { } } -export class ERR_OS_NO_HOMEDIR extends NodeSystemError { - constructor() { - const code = isWindows ? "ENOENT" : "ENOTDIR"; - const ctx: NodeSystemErrorCtx = { - message: "not a directory", - syscall: "home", - code, - errno: isWindows ? osConstants.errno.ENOENT : osConstants.errno.ENOTDIR, - }; - super(code, ctx, "Path is not a directory"); - } -} - export class ERR_HTTP_SOCKET_ASSIGNED extends NodeError { constructor() { super( @@ -2646,11 +2647,30 @@ export function aggregateTwoErrors( } return innerError || outerError; } + +export class NodeAggregateError extends AggregateError { + code: string; + constructor(errors, message) { + super(new SafeArrayIterator(errors), message); + this.code = errors[0]?.code; + } + + get [kIsNodeError]() { + return true; + } + + // deno-lint-ignore adjacent-overload-signatures + get ["constructor"]() { + return AggregateError; + } +} + codes.ERR_IPC_CHANNEL_CLOSED = ERR_IPC_CHANNEL_CLOSED; codes.ERR_INVALID_ARG_TYPE = ERR_INVALID_ARG_TYPE; codes.ERR_INVALID_ARG_VALUE = ERR_INVALID_ARG_VALUE; codes.ERR_OUT_OF_RANGE = ERR_OUT_OF_RANGE; codes.ERR_SOCKET_BAD_PORT = ERR_SOCKET_BAD_PORT; +codes.ERR_SOCKET_CONNECTION_TIMEOUT = ERR_SOCKET_CONNECTION_TIMEOUT; codes.ERR_BUFFER_OUT_OF_BOUNDS = ERR_BUFFER_OUT_OF_BOUNDS; codes.ERR_UNKNOWN_ENCODING = ERR_UNKNOWN_ENCODING; codes.ERR_PARSE_ARGS_INVALID_OPTION_VALUE = ERR_PARSE_ARGS_INVALID_OPTION_VALUE; @@ -2851,6 +2871,7 @@ export default { ERR_INVALID_SYNC_FORK_INPUT, ERR_INVALID_THIS, ERR_INVALID_TUPLE, + ERR_NOT_IMPLEMENTED, ERR_INVALID_URI, ERR_INVALID_URL, ERR_INVALID_URL_SCHEME, diff --git a/ext/node/polyfills/internal/fs/handle.ts b/ext/node/polyfills/internal/fs/handle.ts index fc3a7ae20599a0..9ec0fc97e25642 100644 --- a/ext/node/polyfills/internal/fs/handle.ts +++ b/ext/node/polyfills/internal/fs/handle.ts @@ -6,6 +6,7 @@ import { EventEmitter } from "node:events"; import { Buffer } from "node:buffer"; import { promises, read, write } from "node:fs"; +export type { BigIntStats, Stats } from "ext:deno_node/_fs/_fs_stat.ts"; import { BinaryOptionsArgument, FileOptionsArgument, @@ -141,6 +142,13 @@ export class FileHandle extends EventEmitter { // Note that Deno.close is not async return Promise.resolve(core.close(this.fd)); } + + stat(): Promise; + stat(options: { bigint: false }): Promise; + stat(options: { bigint: true }): Promise; + stat(options?: { bigint: boolean }): Promise { + return fsCall(promises.fstat, this, options); + } } function fsCall(fn, handle, ...args) { @@ -152,7 +160,7 @@ function fsCall(fn, handle, ...args) { }); } - return fn(handle, ...args); + return fn(handle.fd, ...args); } export default { diff --git a/ext/node/polyfills/internal/net.ts b/ext/node/polyfills/internal/net.ts index 144612626f7f1b..a3dcb3ed2159b1 100644 --- a/ext/node/polyfills/internal/net.ts +++ b/ext/node/polyfills/internal/net.ts @@ -95,4 +95,5 @@ export function makeSyncWrite(fd: number) { }; } +export const kReinitializeHandle = Symbol("kReinitializeHandle"); export const normalizedArgsSymbol = Symbol("normalizedArgs"); diff --git a/ext/node/polyfills/internal/util/inspect.mjs b/ext/node/polyfills/internal/util/inspect.mjs index 3a61c387c00dc1..ae797449bfce98 100644 --- a/ext/node/polyfills/internal/util/inspect.mjs +++ b/ext/node/polyfills/internal/util/inspect.mjs @@ -565,6 +565,19 @@ export function stripVTControlCharacters(str) { export function styleText(format, text) { validateString(text, "text"); + + if (Array.isArray(format)) { + for (let i = 0; i < format.length; i++) { + const item = format[i]; + const formatCodes = inspect.colors[item]; + if (formatCodes == null) { + validateOneOf(item, "format", Object.keys(inspect.colors)); + } + text = `\u001b[${formatCodes[0]}m${text}\u001b[${formatCodes[1]}m`; + } + return text; + } + const formatCodes = inspect.colors[format]; if (formatCodes == null) { validateOneOf(format, "format", Object.keys(inspect.colors)); diff --git a/ext/node/polyfills/internal_binding/_timingSafeEqual.ts b/ext/node/polyfills/internal_binding/_timingSafeEqual.ts index ff141fdbfc8aed..559b7685b8bcc3 100644 --- a/ext/node/polyfills/internal_binding/_timingSafeEqual.ts +++ b/ext/node/polyfills/internal_binding/_timingSafeEqual.ts @@ -5,10 +5,11 @@ import { Buffer } from "node:buffer"; -function assert(cond) { - if (!cond) { - throw new Error("assertion failed"); +function toDataView(ab: ArrayBufferLike | ArrayBufferView): DataView { + if (ArrayBuffer.isView(ab)) { + return new DataView(ab.buffer, ab.byteOffset, ab.byteLength); } + return new DataView(ab); } /** Compare to array buffers or data views in a way that timing based attacks @@ -21,13 +22,11 @@ function stdTimingSafeEqual( return false; } if (!(a instanceof DataView)) { - a = new DataView(ArrayBuffer.isView(a) ? a.buffer : a); + a = toDataView(a); } if (!(b instanceof DataView)) { - b = new DataView(ArrayBuffer.isView(b) ? b.buffer : b); + b = toDataView(b); } - assert(a instanceof DataView); - assert(b instanceof DataView); const length = a.byteLength; let out = 0; let i = -1; @@ -41,7 +40,11 @@ export const timingSafeEqual = ( a: Buffer | DataView | ArrayBuffer, b: Buffer | DataView | ArrayBuffer, ): boolean => { - if (a instanceof Buffer) a = new DataView(a.buffer); - if (a instanceof Buffer) b = new DataView(a.buffer); + if (a instanceof Buffer) { + a = new DataView(a.buffer, a.byteOffset, a.byteLength); + } + if (b instanceof Buffer) { + b = new DataView(b.buffer, b.byteOffset, b.byteLength); + } return stdTimingSafeEqual(a, b); }; diff --git a/ext/node/polyfills/internal_binding/http_parser.ts b/ext/node/polyfills/internal_binding/http_parser.ts new file mode 100644 index 00000000000000..bad10d98519edf --- /dev/null +++ b/ext/node/polyfills/internal_binding/http_parser.ts @@ -0,0 +1,160 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +import { primordials } from "ext:core/mod.js"; +import { AsyncWrap } from "ext:deno_node/internal_binding/async_wrap.ts"; + +const { + ObjectDefineProperty, + ObjectEntries, + ObjectSetPrototypeOf, + SafeArrayIterator, +} = primordials; + +export const methods = [ + "DELETE", + "GET", + "HEAD", + "POST", + "PUT", + "CONNECT", + "OPTIONS", + "TRACE", + "COPY", + "LOCK", + "MKCOL", + "MOVE", + "PROPFIND", + "PROPPATCH", + "SEARCH", + "UNLOCK", + "BIND", + "REBIND", + "UNBIND", + "ACL", + "REPORT", + "MKACTIVITY", + "CHECKOUT", + "MERGE", + "M-SEARCH", + "NOTIFY", + "SUBSCRIBE", + "UNSUBSCRIBE", + "PATCH", + "PURGE", + "MKCALENDAR", + "LINK", + "UNLINK", + "SOURCE", + "QUERY", +]; + +export const allMethods = [ + "DELETE", + "GET", + "HEAD", + "POST", + "PUT", + "CONNECT", + "OPTIONS", + "TRACE", + "COPY", + "LOCK", + "MKCOL", + "MOVE", + "PROPFIND", + "PROPPATCH", + "SEARCH", + "UNLOCK", + "BIND", + "REBIND", + "UNBIND", + "ACL", + "REPORT", + "MKACTIVITY", + "CHECKOUT", + "MERGE", + "M-SEARCH", + "NOTIFY", + "SUBSCRIBE", + "UNSUBSCRIBE", + "PATCH", + "PURGE", + "MKCALENDAR", + "LINK", + "UNLINK", + "SOURCE", + "PRI", + "DESCRIBE", + "ANNOUNCE", + "SETUP", + "PLAY", + "PAUSE", + "TEARDOWN", + "GET_PARAMETER", + "SET_PARAMETER", + "REDIRECT", + "RECORD", + "FLUSH", + "QUERY", +]; + +export function HTTPParser() { +} + +ObjectSetPrototypeOf(HTTPParser.prototype, AsyncWrap.prototype); + +function defineProps(obj: object, props: Record) { + for (const entry of new SafeArrayIterator(ObjectEntries(props))) { + ObjectDefineProperty(obj, entry[0], { + __proto__: null, + value: entry[1], + enumerable: true, + writable: true, + configurable: true, + }); + } +} + +defineProps(HTTPParser, { + REQUEST: 1, + RESPONSE: 2, + kOnMessageBegin: 0, + kOnHeaders: 1, + kOnHeadersComplete: 2, + kOnBody: 3, + kOnMessageComplete: 4, + kOnExecute: 5, + kOnTimeout: 6, + kLenientNone: 0, + kLenientHeaders: 1, + kLenientChunkedLength: 2, + kLenientKeepAlive: 4, + kLenientTransferEncoding: 8, + kLenientVersion: 16, + kLenientDataAfterClose: 32, + kLenientOptionalLFAfterCR: 64, + kLenientOptionalCRLFAfterChunk: 128, + kLenientOptionalCRBeforeLF: 256, + kLenientSpacesAfterChunkSize: 512, + kLenientAll: 1023, +}); diff --git a/ext/node/polyfills/internal_binding/mod.ts b/ext/node/polyfills/internal_binding/mod.ts index f2d7f55bc71428..ebbfc629f1c9ec 100644 --- a/ext/node/polyfills/internal_binding/mod.ts +++ b/ext/node/polyfills/internal_binding/mod.ts @@ -17,6 +17,7 @@ import * as types from "ext:deno_node/internal_binding/types.ts"; import * as udpWrap from "ext:deno_node/internal_binding/udp_wrap.ts"; import * as util from "ext:deno_node/internal_binding/util.ts"; import * as uv from "ext:deno_node/internal_binding/uv.ts"; +import * as httpParser from "ext:deno_node/internal_binding/http_parser.ts"; const modules = { "async_wrap": asyncWrap, @@ -32,7 +33,7 @@ const modules = { "fs_dir": {}, "fs_event_wrap": {}, "heap_utils": {}, - "http_parser": {}, + "http_parser": httpParser, icu: {}, inspector: {}, "js_stream": {}, diff --git a/ext/node/polyfills/internal_binding/stream_wrap.ts b/ext/node/polyfills/internal_binding/stream_wrap.ts index 7aea83d6f5899c..19c9357ce8287d 100644 --- a/ext/node/polyfills/internal_binding/stream_wrap.ts +++ b/ext/node/polyfills/internal_binding/stream_wrap.ts @@ -320,8 +320,16 @@ export class LibuvStreamWrap extends HandleWrap { /** Internal method for reading from the attached stream. */ async #read() { let buf = this.#buf; + let nread: number | null; const ridBefore = this[kStreamBaseField]![internalRidSymbol]; + + if (this.upgrading) { + // Starting an upgrade, stop reading. Upgrading will resume reading. + this.readStop(); + return; + } + try { nread = await this[kStreamBaseField]!.read(buf); } catch (e) { @@ -382,6 +390,11 @@ export class LibuvStreamWrap extends HandleWrap { const ridBefore = this[kStreamBaseField]![internalRidSymbol]; + if (this.upgrading) { + // There is an upgrade in progress, queue the write request. + await this.upgrading; + } + let nwritten = 0; try { // TODO(crowlKats): duplicate from runtime/js/13_buffer.js @@ -400,7 +413,6 @@ export class LibuvStreamWrap extends HandleWrap { } let status: number; - // TODO(cmorten): map err to status codes if ( e instanceof Deno.errors.BadResource || diff --git a/ext/node/polyfills/internal_binding/tcp_wrap.ts b/ext/node/polyfills/internal_binding/tcp_wrap.ts index 973a1d1c01bba2..d9f1c5356a0fda 100644 --- a/ext/node/polyfills/internal_binding/tcp_wrap.ts +++ b/ext/node/polyfills/internal_binding/tcp_wrap.ts @@ -299,8 +299,10 @@ export class TCP extends ConnectionWrap { * @param noDelay * @return An error status code. */ - setNoDelay(_noDelay: boolean): number { - // TODO(bnoordhuis) https://github.com/denoland/deno/pull/13103 + setNoDelay(noDelay: boolean): number { + if (this[kStreamBaseField] && "setNoDelay" in this[kStreamBaseField]) { + this[kStreamBaseField].setNoDelay(noDelay); + } return 0; } diff --git a/ext/node/polyfills/internal_binding/uv.ts b/ext/node/polyfills/internal_binding/uv.ts index aa468a0a583b07..6cd70a7e85288e 100644 --- a/ext/node/polyfills/internal_binding/uv.ts +++ b/ext/node/polyfills/internal_binding/uv.ts @@ -530,10 +530,12 @@ export function mapSysErrnoToUvErrno(sysErrno: number): number { export const UV_EAI_MEMORY = codeMap.get("EAI_MEMORY")!; export const UV_EBADF = codeMap.get("EBADF")!; +export const UV_ECANCELED = codeMap.get("ECANCELED")!; export const UV_EEXIST = codeMap.get("EEXIST"); export const UV_EINVAL = codeMap.get("EINVAL")!; export const UV_ENOENT = codeMap.get("ENOENT"); export const UV_ENOTSOCK = codeMap.get("ENOTSOCK")!; +export const UV_ETIMEDOUT = codeMap.get("ETIMEDOUT")!; export const UV_UNKNOWN = codeMap.get("UNKNOWN")!; export function errname(errno: number): string { diff --git a/ext/node/polyfills/net.ts b/ext/node/polyfills/net.ts index 6625ce7b50044e..2b011251905212 100644 --- a/ext/node/polyfills/net.ts +++ b/ext/node/polyfills/net.ts @@ -31,6 +31,7 @@ import { isIP, isIPv4, isIPv6, + kReinitializeHandle, normalizedArgsSymbol, } from "ext:deno_node/internal/net.ts"; import { Duplex } from "node:stream"; @@ -50,9 +51,11 @@ import { ERR_SERVER_ALREADY_LISTEN, ERR_SERVER_NOT_RUNNING, ERR_SOCKET_CLOSED, + ERR_SOCKET_CONNECTION_TIMEOUT, errnoException, exceptionWithHostPort, genericNodeError, + NodeAggregateError, uvExceptionWithHostPort, } from "ext:deno_node/internal/errors.ts"; import type { ErrnoException } from "ext:deno_node/internal/errors.ts"; @@ -80,6 +83,7 @@ import { Buffer } from "node:buffer"; import type { LookupOneOptions } from "ext:deno_node/internal/dns/utils.ts"; import { validateAbortSignal, + validateBoolean, validateFunction, validateInt32, validateNumber, @@ -100,13 +104,25 @@ import { ShutdownWrap } from "ext:deno_node/internal_binding/stream_wrap.ts"; import { assert } from "ext:deno_node/_util/asserts.ts"; import { isWindows } from "ext:deno_node/_util/os.ts"; import { ADDRCONFIG, lookup as dnsLookup } from "node:dns"; -import { codeMap } from "ext:deno_node/internal_binding/uv.ts"; +import { + codeMap, + UV_ECANCELED, + UV_ETIMEDOUT, +} from "ext:deno_node/internal_binding/uv.ts"; import { guessHandleType } from "ext:deno_node/internal_binding/util.ts"; import { debuglog } from "ext:deno_node/internal/util/debuglog.ts"; import type { DuplexOptions } from "ext:deno_node/_stream.d.ts"; import type { BufferEncoding } from "ext:deno_node/_global.d.ts"; import type { Abortable } from "ext:deno_node/_events.d.ts"; import { channel } from "node:diagnostics_channel"; +import { primordials } from "ext:core/mod.js"; + +const { + ArrayPrototypeIncludes, + ArrayPrototypePush, + FunctionPrototypeBind, + MathMax, +} = primordials; let debug = debuglog("net", (fn) => { debug = fn; @@ -120,6 +136,9 @@ const kBytesWritten = Symbol("kBytesWritten"); const DEFAULT_IPV4_ADDR = "0.0.0.0"; const DEFAULT_IPV6_ADDR = "::"; +let autoSelectFamilyDefault = true; +let autoSelectFamilyAttemptTimeoutDefault = 250; + type Handle = TCP | Pipe; interface HandleOptions { @@ -214,6 +233,8 @@ interface TcpSocketConnectOptions extends ConnectOptions { hints?: number; family?: number; lookup?: LookupFunction; + autoSelectFamily?: boolean | undefined; + autoSelectFamilyAttemptTimeout?: number | undefined; } interface IpcSocketConnectOptions extends ConnectOptions { @@ -316,12 +337,6 @@ export function _normalizeArgs(args: unknown[]): NormalizedArgs { return arr; } -function _isTCPConnectWrap( - req: TCPConnectWrap | PipeConnectWrap, -): req is TCPConnectWrap { - return "localAddress" in req && "localPort" in req; -} - function _afterConnect( status: number, // deno-lint-ignore no-explicit-any @@ -372,7 +387,7 @@ function _afterConnect( socket.connecting = false; let details; - if (_isTCPConnectWrap(req)) { + if (req.localAddress && req.localPort) { details = req.localAddress + ":" + req.localPort; } @@ -384,7 +399,7 @@ function _afterConnect( details, ); - if (_isTCPConnectWrap(req)) { + if (details) { ex.localAddress = req.localAddress; ex.localPort = req.localPort; } @@ -393,6 +408,107 @@ function _afterConnect( } } +function _createConnectionError(req, status) { + let details; + + if (req.localAddress && req.localPort) { + details = req.localAddress + ":" + req.localPort; + } + + const ex = exceptionWithHostPort( + status, + "connect", + req.address, + req.port, + details, + ); + if (details) { + ex.localAddress = req.localAddress; + ex.localPort = req.localPort; + } + + return ex; +} + +function _afterConnectMultiple( + context, + current, + status, + handle, + req, + readable, + writable, +) { + debug( + "connect/multiple: connection attempt to %s:%s completed with status %s", + req.address, + req.port, + status, + ); + + // Make sure another connection is not spawned + clearTimeout(context[kTimeout]); + + // One of the connection has completed and correctly dispatched but after timeout, ignore this one + if (status === 0 && current !== context.current - 1) { + debug( + "connect/multiple: ignoring successful but timedout connection to %s:%s", + req.address, + req.port, + ); + handle.close(); + return; + } + + const self = context.socket; + + // Some error occurred, add to the list of exceptions + if (status !== 0) { + const ex = _createConnectionError(req, status); + ArrayPrototypePush(context.errors, ex); + + self.emit( + "connectionAttemptFailed", + req.address, + req.port, + req.addressType, + ex, + ); + + // Try the next address, unless we were aborted + if (context.socket.connecting) { + _internalConnectMultiple(context, status === UV_ECANCELED); + } + + return; + } + + _afterConnect(status, self._handle, req, readable, writable); +} + +function _internalConnectMultipleTimeout(context, req, handle) { + debug( + "connect/multiple: connection to %s:%s timed out", + req.address, + req.port, + ); + context.socket.emit( + "connectionAttemptTimeout", + req.address, + req.port, + req.addressType, + ); + + req.oncomplete = undefined; + ArrayPrototypePush(context.errors, _createConnectionError(req, UV_ETIMEDOUT)); + handle.close(); + + // Try the next address, unless we were aborted + if (context.socket.connecting) { + _internalConnectMultiple(context); + } +} + function _checkBindError(err: number, port: number, handle: TCP) { // EADDRINUSE may not be reported until we call `listen()` or `connect()`. // To complicate matters, a failed `bind()` followed by `listen()` or `connect()` @@ -495,6 +611,131 @@ function _internalConnect( } } +function _internalConnectMultiple(context, canceled?: boolean) { + clearTimeout(context[kTimeout]); + const self = context.socket; + + // We were requested to abort. Stop all operations + if (self._aborted) { + return; + } + + // All connections have been tried without success, destroy with error + if (canceled || context.current === context.addresses.length) { + if (context.errors.length === 0) { + self.destroy(new ERR_SOCKET_CONNECTION_TIMEOUT()); + return; + } + + self.destroy(new NodeAggregateError(context.errors)); + return; + } + + assert(self.connecting); + + const current = context.current++; + + if (current > 0) { + self[kReinitializeHandle](new TCP(TCPConstants.SOCKET)); + } + + const { localPort, port, flags } = context; + const { address, family: addressType } = context.addresses[current]; + let localAddress; + let err; + + if (localPort) { + if (addressType === 4) { + localAddress = DEFAULT_IPV4_ADDR; + err = self._handle.bind(localAddress, localPort); + } else { // addressType === 6 + localAddress = DEFAULT_IPV6_ADDR; + err = self._handle.bind6(localAddress, localPort, flags); + } + + debug( + "connect/multiple: binding to localAddress: %s and localPort: %d (addressType: %d)", + localAddress, + localPort, + addressType, + ); + + err = _checkBindError(err, localPort, self._handle); + if (err) { + ArrayPrototypePush( + context.errors, + exceptionWithHostPort(err, "bind", localAddress, localPort), + ); + _internalConnectMultiple(context); + return; + } + } + + debug( + "connect/multiple: attempting to connect to %s:%d (addressType: %d)", + address, + port, + addressType, + ); + self.emit("connectionAttempt", address, port, addressType); + + const req = new TCPConnectWrap(); + req.oncomplete = FunctionPrototypeBind( + _afterConnectMultiple, + undefined, + context, + current, + ); + req.address = address; + req.port = port; + req.localAddress = localAddress; + req.localPort = localPort; + req.addressType = addressType; + + ArrayPrototypePush( + self.autoSelectFamilyAttemptedAddresses, + `${address}:${port}`, + ); + + if (addressType === 4) { + err = self._handle.connect(req, address, port); + } else { + err = self._handle.connect6(req, address, port); + } + + if (err) { + const sockname = self._getsockname(); + let details; + + if (sockname) { + details = sockname.address + ":" + sockname.port; + } + + const ex = exceptionWithHostPort(err, "connect", address, port, details); + ArrayPrototypePush(context.errors, ex); + + self.emit("connectionAttemptFailed", address, port, addressType, ex); + _internalConnectMultiple(context); + return; + } + + if (current < context.addresses.length - 1) { + debug( + "connect/multiple: setting the attempt timeout to %d ms", + context.timeout, + ); + + // If the attempt has not returned an error, start the connection timer + context[kTimeout] = setTimeout( + _internalConnectMultipleTimeout, + context.timeout, + context, + req, + self._handle, + ); + } +} + // Provide a better error message when we call end() as a result // of the other side sending a FIN. The standard "write after end" // is overly vague, and makes it seem like the user's code is to blame. @@ -597,7 +838,7 @@ function _lookupAndConnect( ) { const { localAddress, localPort } = options; const host = options.host || "localhost"; - let { port } = options; + let { port, autoSelectFamilyAttemptTimeout, autoSelectFamily } = options; if (localAddress && !isIP(localAddress)) { throw new ERR_INVALID_IP_ADDRESS(localAddress); @@ -621,6 +862,22 @@ function _lookupAndConnect( port |= 0; + if (autoSelectFamily != null) { + validateBoolean(autoSelectFamily, "options.autoSelectFamily"); + } else { + autoSelectFamily = autoSelectFamilyDefault; + } + + if (autoSelectFamilyAttemptTimeout !== undefined) { + validateInt32(autoSelectFamilyAttemptTimeout); + + if (autoSelectFamilyAttemptTimeout < 10) { + autoSelectFamilyAttemptTimeout = 10; + } + } else { + autoSelectFamilyAttemptTimeout = autoSelectFamilyAttemptTimeoutDefault; + } + // If host is an IP, skip performing a lookup const addressType = isIP(host); if (addressType) { @@ -649,6 +906,7 @@ function _lookupAndConnect( const dnsOpts = { family: options.family, hints: options.hints || 0, + all: false, }; if ( @@ -665,6 +923,31 @@ function _lookupAndConnect( self._host = host; const lookup = options.lookup || dnsLookup; + if ( + dnsOpts.family !== 4 && dnsOpts.family !== 6 && !localAddress && + autoSelectFamily + ) { + debug("connect: autodetecting"); + + dnsOpts.all = true; + defaultTriggerAsyncIdScope(self[asyncIdSymbol], function () { + _lookupAndConnectMultiple( + self, + asyncIdSymbol, + lookup, + host, + options, + dnsOpts, + port, + localAddress, + localPort, + autoSelectFamilyAttemptTimeout, + ); + }); + + return; + } + defaultTriggerAsyncIdScope(self[asyncIdSymbol], function () { lookup( host, @@ -719,6 +1002,143 @@ function _lookupAndConnect( }); } +function _lookupAndConnectMultiple( + self: Socket, + asyncIdSymbol: number, + // deno-lint-ignore no-explicit-any + lookup: any, + host: string, + options: TcpSocketConnectOptions, + dnsopts, + port: number, + localAddress: string, + localPort: number, + timeout: number | undefined, +) { + defaultTriggerAsyncIdScope(self[asyncIdSymbol], function emitLookup() { + lookup(host, dnsopts, function emitLookup(err, addresses) { + // It's possible we were destroyed while looking this up. + // XXX it would be great if we could cancel the promise returned by + // the look up. + if (!self.connecting) { + return; + } else if (err) { + self.emit("lookup", err, undefined, undefined, host); + + // net.createConnection() creates a net.Socket object and immediately + // calls net.Socket.connect() on it (that's us). There are no event + // listeners registered yet so defer the error event to the next tick. + nextTick(_connectErrorNT, self, err); + return; + } + + // Filter addresses by only keeping the one which are either IPv4 or IPV6. + // The first valid address determines which group has preference on the + // alternate family sorting which happens later. + const validAddresses = [[], []]; + const validIps = [[], []]; + let destinations; + for (let i = 0, l = addresses.length; i < l; i++) { + const address = addresses[i]; + const { address: ip, family: addressType } = address; + self.emit("lookup", err, ip, addressType, host); + // It's possible we were destroyed while looking this up. + if (!self.connecting) { + return; + } + if (isIP(ip) && (addressType === 4 || addressType === 6)) { + destinations ||= addressType === 6 ? { 6: 0, 4: 1 } : { 4: 0, 6: 1 }; + + const destination = destinations[addressType]; + + // Only try an address once + if (!ArrayPrototypeIncludes(validIps[destination], ip)) { + ArrayPrototypePush(validAddresses[destination], address); + ArrayPrototypePush(validIps[destination], ip); + } + } + } + + // When no AAAA or A records are available, fail on the first one + if (!validAddresses[0].length && !validAddresses[1].length) { + const { address: firstIp, family: firstAddressType } = addresses[0]; + + if (!isIP(firstIp)) { + err = new ERR_INVALID_IP_ADDRESS(firstIp); + nextTick(_connectErrorNT, self, err); + } else if (firstAddressType !== 4 && firstAddressType !== 6) { + err = new ERR_INVALID_ADDRESS_FAMILY( + firstAddressType, + options.host, + options.port, + ); + nextTick(_connectErrorNT, self, err); + } + + return; + } + + // Sort addresses alternating families + const toAttempt = []; + for ( + let i = 0, + l = MathMax(validAddresses[0].length, validAddresses[1].length); + i < l; + i++ + ) { + if (i in validAddresses[0]) { + ArrayPrototypePush(toAttempt, validAddresses[0][i]); + } + if (i in validAddresses[1]) { + ArrayPrototypePush(toAttempt, validAddresses[1][i]); + } + } + + if (toAttempt.length === 1) { + debug( + "connect/multiple: only one address found, switching back to single connection", + ); + const { address: ip, family: addressType } = toAttempt[0]; + + self._unrefTimer(); + defaultTriggerAsyncIdScope( + self[asyncIdSymbol], + _internalConnect, + self, + ip, + port, + addressType, + localAddress, + localPort, + ); + + return; + } + + self.autoSelectFamilyAttemptedAddresses = []; + debug("connect/multiple: will try the following addresses", toAttempt); + + const context = { + socket: self, + addresses: toAttempt, + current: 0, + port, + localPort, + timeout, + [kTimeout]: null, + errors: [], + }; + + self._unrefTimer(); + defaultTriggerAsyncIdScope( + self[asyncIdSymbol], + _internalConnectMultiple, + context, + ); + }); + }); +} + function _afterShutdown(this: ShutdownWrap) { // deno-lint-ignore no-explicit-any const self: any = this.handle[ownerSymbol]; @@ -777,6 +1197,7 @@ export class Socket extends Duplex { _host: string | null = null; // deno-lint-ignore no-explicit-any _parent: any = null; + autoSelectFamilyAttemptedAddresses: AddressInfo[] | undefined = undefined; constructor(options: SocketOptions | number) { if (typeof options === "number") { @@ -951,7 +1372,6 @@ export class Socket extends Duplex { */ override pause(): this { if ( - this[kBuffer] && !this.connecting && this._handle && this._handle.reading @@ -1547,6 +1967,16 @@ export class Socket extends Duplex { set _handle(v: Handle | null) { this[kHandle] = v; } + + // deno-lint-ignore no-explicit-any + [kReinitializeHandle](handle: any) { + this._handle?.close(); + + this._handle = handle; + this._handle[ownerSymbol] = this; + + _initSocketHandle(this); + } } export const Stream = Socket; @@ -1594,6 +2024,33 @@ export function connect(...args: unknown[]) { export const createConnection = connect; +/** https://docs.deno.com/api/node/net/#namespace_getdefaultautoselectfamily */ +export function getDefaultAutoSelectFamily() { + return autoSelectFamilyDefault; +} + +/** https://docs.deno.com/api/node/net/#namespace_setdefaultautoselectfamily */ +export function setDefaultAutoSelectFamily(value: boolean) { + validateBoolean(value, "value"); + autoSelectFamilyDefault = value; +} + +/** https://docs.deno.com/api/node/net/#namespace_getdefaultautoselectfamilyattempttimeout */ +export function getDefaultAutoSelectFamilyAttemptTimeout() { + return autoSelectFamilyAttemptTimeoutDefault; +} + +/** https://docs.deno.com/api/node/net/#namespace_setdefaultautoselectfamilyattempttimeout */ +export function setDefaultAutoSelectFamilyAttemptTimeout(value: number) { + validateInt32(value, "value", 1); + + if (value < 10) { + value = 10; + } + + autoSelectFamilyAttemptTimeoutDefault = value; +} + export interface ListenOptions extends Abortable { fd?: number; port?: number | undefined; @@ -2479,15 +2936,19 @@ export { BlockList, isIP, isIPv4, isIPv6, SocketAddress }; export default { _createServerHandle, _normalizeArgs, - isIP, - isIPv4, - isIPv6, BlockList, - SocketAddress, connect, createConnection, createServer, + getDefaultAutoSelectFamily, + getDefaultAutoSelectFamilyAttemptTimeout, + isIP, + isIPv4, + isIPv6, Server, + setDefaultAutoSelectFamily, + setDefaultAutoSelectFamilyAttemptTimeout, Socket, + SocketAddress, Stream, }; diff --git a/ext/node/polyfills/os.ts b/ext/node/polyfills/os.ts index e47e8679ec12b1..edc89ed2c3f113 100644 --- a/ext/node/polyfills/os.ts +++ b/ext/node/polyfills/os.ts @@ -28,16 +28,17 @@ import { op_homedir, op_node_os_get_priority, op_node_os_set_priority, - op_node_os_username, + op_node_os_user_info, } from "ext:core/ops"; import { validateIntegerRange } from "ext:deno_node/_utils.ts"; import process from "node:process"; import { isWindows } from "ext:deno_node/_util/os.ts"; -import { ERR_OS_NO_HOMEDIR } from "ext:deno_node/internal/errors.ts"; import { os } from "ext:deno_node/internal_binding/constants.ts"; import { osUptime } from "ext:runtime/30_os.js"; import { Buffer } from "ext:deno_node/internal/buffer.mjs"; +import { primordials } from "ext:core/mod.js"; +const { StringPrototypeEndsWith, StringPrototypeSlice } = primordials; export const constants = os; @@ -136,6 +137,8 @@ export function arch(): string { (uptime as any)[Symbol.toPrimitive] = (): number => uptime(); // deno-lint-ignore no-explicit-any (machine as any)[Symbol.toPrimitive] = (): string => machine(); +// deno-lint-ignore no-explicit-any +(tmpdir as any)[Symbol.toPrimitive] = (): string | null => tmpdir(); export function cpus(): CPUCoreInfo[] { return op_cpus(); @@ -268,26 +271,27 @@ export function setPriority(pid: number, priority?: number) { export function tmpdir(): string | null { /* This follows the node js implementation, but has a few differences: - * On windows, if none of the environment variables are defined, - we return null. - * On unix we use a plain Deno.env.get, instead of safeGetenv, + * We use a plain Deno.env.get, instead of safeGetenv, which special cases setuid binaries. - * Node removes a single trailing / or \, we remove all. */ if (isWindows) { - const temp = Deno.env.get("TEMP") || Deno.env.get("TMP"); - if (temp) { - return temp.replace(/(? 1 && StringPrototypeEndsWith(temp, "\\") && + !StringPrototypeEndsWith(temp, ":\\") + ) { + temp = StringPrototypeSlice(temp, 0, -1); } - return null; + + return temp; } else { // !isWindows - const temp = Deno.env.get("TMPDIR") || Deno.env.get("TMP") || + let temp = Deno.env.get("TMPDIR") || Deno.env.get("TMP") || Deno.env.get("TEMP") || "/tmp"; - return temp.replace(/(? 1 && StringPrototypeEndsWith(temp, "/")) { + temp = StringPrototypeSlice(temp, 0, -1); + } + return temp; } } @@ -320,7 +324,6 @@ export function uptime(): number { return osUptime(); } -/** Not yet implemented */ export function userInfo( options: UserInfoOptions = { encoding: "utf-8" }, ): UserInfo { @@ -331,20 +334,10 @@ export function userInfo( uid = -1; gid = -1; } - - // TODO(@crowlKats): figure out how to do this correctly: - // The value of homedir returned by os.userInfo() is provided by the operating system. - // This differs from the result of os.homedir(), which queries environment - // variables for the home directory before falling back to the operating system response. - let _homedir = homedir(); - if (!_homedir) { - throw new ERR_OS_NO_HOMEDIR(); - } - let shell = isWindows ? null : (Deno.env.get("SHELL") || null); - let username = op_node_os_username(); + let { username, homedir, shell } = op_node_os_user_info(uid); if (options?.encoding === "buffer") { - _homedir = _homedir ? Buffer.from(_homedir) : _homedir; + homedir = homedir ? Buffer.from(homedir) : homedir; shell = shell ? Buffer.from(shell) : shell; username = Buffer.from(username); } @@ -352,7 +345,7 @@ export function userInfo( return { uid, gid, - homedir: _homedir, + homedir, shell, username, }; diff --git a/ext/node/polyfills/perf_hooks.ts b/ext/node/polyfills/perf_hooks.ts index d92b925b5feefc..ec76b3ce2d89c9 100644 --- a/ext/node/polyfills/perf_hooks.ts +++ b/ext/node/polyfills/perf_hooks.ts @@ -8,6 +8,7 @@ import { performance as shimPerformance, PerformanceEntry, } from "ext:deno_web/15_performance.js"; +import { EldHistogram } from "ext:core/ops"; class PerformanceObserver { static supportedEntryTypes: string[] = []; @@ -89,10 +90,11 @@ const performance: ) => shimPerformance.dispatchEvent(...args), }; -const monitorEventLoopDelay = () => - notImplemented( - "monitorEventLoopDelay from performance", - ); +function monitorEventLoopDelay(options = {}) { + const { resolution = 10 } = options; + + return new EldHistogram(resolution); +} export default { performance, diff --git a/ext/node/polyfills/process.ts b/ext/node/polyfills/process.ts index 3dc6ce61aabd7c..647376d5cf0147 100644 --- a/ext/node/polyfills/process.ts +++ b/ext/node/polyfills/process.ts @@ -15,7 +15,7 @@ import { import { warnNotImplemented } from "ext:deno_node/_utils.ts"; import { EventEmitter } from "node:events"; -import Module from "node:module"; +import Module, { getBuiltinModule } from "node:module"; import { report } from "ext:deno_node/internal/process/report.ts"; import { validateString } from "ext:deno_node/internal/validators.mjs"; import { @@ -38,7 +38,15 @@ import { versions, } from "ext:deno_node/_process/process.ts"; import { _exiting } from "ext:deno_node/_process/exiting.ts"; -export { _nextTick as nextTick, chdir, cwd, env, version, versions }; +export { + _nextTick as nextTick, + chdir, + cwd, + env, + getBuiltinModule, + version, + versions, +}; import { createWritableStdioStream, initStdin, @@ -520,9 +528,7 @@ Process.prototype.on = function ( } else if ( event !== "SIGBREAK" && event !== "SIGINT" && Deno.build.os === "windows" ) { - // Ignores all signals except SIGBREAK and SIGINT on windows. - // deno-lint-ignore no-console - console.warn(`Ignoring signal "${event}" on Windows`); + // TODO(#26331): Ignores all signals except SIGBREAK and SIGINT on windows. } else { EventEmitter.prototype.on.call(this, event, listener); Deno.addSignalListener(event as Deno.Signal, listener); @@ -730,6 +736,8 @@ Process.prototype.getegid = getegid; /** This method is removed on Windows */ Process.prototype.geteuid = geteuid; +Process.prototype.getBuiltinModule = getBuiltinModule; + // TODO(kt3k): Implement this when we added -e option to node compat mode Process.prototype._eval = undefined; @@ -911,7 +919,7 @@ Object.defineProperty(argv, "1", { if (Deno.mainModule?.startsWith("file:")) { return pathFromURL(new URL(Deno.mainModule)); } else { - return join(Deno.cwd(), "$deno$node.js"); + return join(Deno.cwd(), "$deno$node.mjs"); } }, }); diff --git a/ext/node/polyfills/timers.ts b/ext/node/polyfills/timers.ts index 02f69466eef615..fa5f7a204255be 100644 --- a/ext/node/polyfills/timers.ts +++ b/ext/node/polyfills/timers.ts @@ -15,10 +15,16 @@ import { setUnrefTimeout, Timeout, } from "ext:deno_node/internal/timers.mjs"; -import { validateFunction } from "ext:deno_node/internal/validators.mjs"; +import { + validateAbortSignal, + validateBoolean, + validateFunction, + validateObject, +} from "ext:deno_node/internal/validators.mjs"; import { promisify } from "ext:deno_node/internal/util.mjs"; export { setUnrefTimeout } from "ext:deno_node/internal/timers.mjs"; import * as timers from "ext:deno_web/02_timers.js"; +import { AbortError } from "ext:deno_node/internal/errors.ts"; const clearTimeout_ = timers.clearTimeout; const clearInterval_ = timers.clearInterval; @@ -48,7 +54,7 @@ export function clearTimeout(timeout?: Timeout | number) { const id = +timeout; const timer = MapPrototypeGet(activeTimers, id); if (timer) { - timeout._destroyed = true; + timer._destroyed = true; MapPrototypeDelete(activeTimers, id); } clearTimeout_(id); @@ -68,7 +74,7 @@ export function clearInterval(timeout?: Timeout | number | string) { const id = +timeout; const timer = MapPrototypeGet(activeTimers, id); if (timer) { - timeout._destroyed = true; + timer._destroyed = true; MapPrototypeDelete(activeTimers, id); } clearInterval_(id); @@ -89,10 +95,88 @@ export function clearImmediate(immediate: Immediate) { clearTimeout_(immediate._immediateId); } +async function* setIntervalAsync( + after: number, + value: number, + options: { signal?: AbortSignal; ref?: boolean } = { __proto__: null }, +) { + validateObject(options, "options"); + + if (typeof options?.signal !== "undefined") { + validateAbortSignal(options.signal, "options.signal"); + } + + if (typeof options?.ref !== "undefined") { + validateBoolean(options.ref, "options.ref"); + } + + const { signal, ref = true } = options; + + if (signal?.aborted) { + throw new AbortError(undefined, { cause: signal?.reason }); + } + + let onCancel: (() => void) | undefined = undefined; + let interval: Timeout | undefined = undefined; + try { + let notYielded = 0; + let callback: ((value?: object) => void) | undefined = undefined; + let rejectCallback: ((message?: string) => void) | undefined = undefined; + interval = new Timeout( + () => { + notYielded++; + if (callback) { + callback(); + callback = undefined; + rejectCallback = undefined; + } + }, + after, + [], + true, + ref, + ); + if (signal) { + onCancel = () => { + clearInterval(interval); + if (rejectCallback) { + rejectCallback(signal.reason); + callback = undefined; + rejectCallback = undefined; + } + }; + signal.addEventListener("abort", onCancel, { once: true }); + } + while (!signal?.aborted) { + if (notYielded === 0) { + await new Promise((resolve: () => void, reject: () => void) => { + callback = resolve; + rejectCallback = reject; + }); + } + for (; notYielded > 0; notYielded--) { + yield value; + } + } + } catch (error) { + if (signal?.aborted) { + throw new AbortError(undefined, { cause: signal?.reason }); + } + throw error; + } finally { + if (interval) { + clearInterval(interval); + } + if (onCancel) { + signal?.removeEventListener("abort", onCancel); + } + } +} + export const promises = { setTimeout: promisify(setTimeout), setImmediate: promisify(setImmediate), - setInterval: promisify(setInterval), + setInterval: setIntervalAsync, }; promises.scheduler = { diff --git a/ext/node/polyfills/util.ts b/ext/node/polyfills/util.ts index 586fae17e9b28b..d82b288b037871 100644 --- a/ext/node/polyfills/util.ts +++ b/ext/node/polyfills/util.ts @@ -39,6 +39,7 @@ import { formatWithOptions, inspect, stripVTControlCharacters, + styleText, } from "ext:deno_node/internal/util/inspect.mjs"; import { codes } from "ext:deno_node/internal/error_codes.ts"; import types from "node:util/types"; @@ -63,6 +64,7 @@ export { parseArgs, promisify, stripVTControlCharacters, + styleText, types, }; @@ -354,4 +356,5 @@ export default { debuglog, debug: debuglog, isDeepStrictEqual, + styleText, }; diff --git a/ext/node/polyfills/v8.ts b/ext/node/polyfills/v8.ts index 5849f3ccc94afd..9df199865e43e9 100644 --- a/ext/node/polyfills/v8.ts +++ b/ext/node/polyfills/v8.ts @@ -313,20 +313,6 @@ export class DefaultDeserializer extends Deserializer { ); } } -export const promiseHooks = { - onInit() { - notImplemented("v8.promiseHooks.onInit"); - }, - onSettled() { - notImplemented("v8.promiseHooks.onSetttled"); - }, - onBefore() { - notImplemented("v8.promiseHooks.onBefore"); - }, - createHook() { - notImplemented("v8.promiseHooks.createHook"); - }, -}; export default { cachedDataVersionTag, getHeapCodeStatistics, @@ -343,5 +329,4 @@ export default { Deserializer, DefaultSerializer, DefaultDeserializer, - promiseHooks, }; diff --git a/ext/node/polyfills/vm.js b/ext/node/polyfills/vm.js index 183ddad2f4d118..b64c847c586af6 100644 --- a/ext/node/polyfills/vm.js +++ b/ext/node/polyfills/vm.js @@ -182,6 +182,7 @@ function getContextOptions(options) { let defaultContextNameIndex = 1; export function createContext( + // deno-lint-ignore prefer-primordials contextObject = {}, options = { __proto__: null }, ) { diff --git a/ext/node/polyfills/worker_threads.ts b/ext/node/polyfills/worker_threads.ts index 5ff4446f732fa7..d4b75fb30c2c3b 100644 --- a/ext/node/polyfills/worker_threads.ts +++ b/ext/node/polyfills/worker_threads.ts @@ -302,8 +302,8 @@ class NodeWorker extends EventEmitter { if (this.#status !== "TERMINATED") { this.#status = "TERMINATED"; op_host_terminate_worker(this.#id); + this.emit("exit", 0); } - this.emit("exit", 0); return PromiseResolve(0); } @@ -422,7 +422,11 @@ internals.__initWorkerThreads = ( parentPort.once = function (this: ParentPort, name, listener) { // deno-lint-ignore no-explicit-any - const _listener = (ev: any) => listener(ev.data); + const _listener = (ev: any) => { + const message = ev.data; + patchMessagePortIfFound(message); + return listener(message); + }; listeners.set(listener, _listener); this.addEventListener(name, _listener); return this; @@ -494,7 +498,9 @@ export function receiveMessageOnPort(port: MessagePort): object | undefined { port[MessagePortReceiveMessageOnPortSymbol] = true; const data = op_message_port_recv_message_sync(port[MessagePortIdSymbol]); if (data === null) return undefined; - return { message: deserializeJsMessageData(data)[0] }; + const message = deserializeJsMessageData(data)[0]; + patchMessagePortIfFound(message); + return { message }; } class NodeMessageChannel { diff --git a/ext/node/polyfills/zlib.ts b/ext/node/polyfills/zlib.ts index 3fe5f8bbd9d3a3..6e5d02b5be9bfb 100644 --- a/ext/node/polyfills/zlib.ts +++ b/ext/node/polyfills/zlib.ts @@ -40,6 +40,58 @@ import { createBrotliCompress, createBrotliDecompress, } from "ext:deno_node/_brotli.js"; +import { ERR_INVALID_ARG_TYPE } from "ext:deno_node/internal/errors.ts"; +import { validateUint32 } from "ext:deno_node/internal/validators.mjs"; +import { op_zlib_crc32 } from "ext:core/ops"; +import { core, primordials } from "ext:core/mod.js"; +import { TextEncoder } from "ext:deno_web/08_text_encoding.js"; +const { + Uint8Array, + TypedArrayPrototypeGetBuffer, + TypedArrayPrototypeGetByteLength, + TypedArrayPrototypeGetByteOffset, + DataViewPrototypeGetBuffer, + DataViewPrototypeGetByteLength, + DataViewPrototypeGetByteOffset, +} = primordials; +const { isTypedArray, isDataView } = core; + +const enc = new TextEncoder(); +const toU8 = (input) => { + if (typeof input === "string") { + return enc.encode(input); + } + + if (isTypedArray(input)) { + return new Uint8Array( + TypedArrayPrototypeGetBuffer(input), + TypedArrayPrototypeGetByteOffset(input), + TypedArrayPrototypeGetByteLength(input), + ); + } else if (isDataView(input)) { + return new Uint8Array( + DataViewPrototypeGetBuffer(input), + DataViewPrototypeGetByteOffset(input), + DataViewPrototypeGetByteLength(input), + ); + } + + return input; +}; + +export function crc32(data, value = 0) { + if (typeof data !== "string" && !isArrayBufferView(data)) { + throw new ERR_INVALID_ARG_TYPE("data", [ + "Buffer", + "TypedArray", + "DataView", + "string", + ], data); + } + validateUint32(value, "value"); + + return op_zlib_crc32(toU8(data), value); +} export class Options { constructor() { @@ -87,6 +139,7 @@ export default { BrotliOptions, codes, constants, + crc32, createBrotliCompress, createBrotliDecompress, createDeflate, diff --git a/ext/node_resolver/Cargo.toml b/ext/node_resolver/Cargo.toml deleted file mode 100644 index 1042045692b419..00000000000000 --- a/ext/node_resolver/Cargo.toml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -[package] -name = "node_resolver" -version = "0.7.0" -authors.workspace = true -edition.workspace = true -license.workspace = true -readme = "README.md" -repository.workspace = true -description = "Node.js module resolution algorithm used in Deno" - -[lib] -path = "lib.rs" - -[features] -sync = ["deno_package_json/sync"] - -[dependencies] -anyhow.workspace = true -async-trait.workspace = true -deno_media_type.workspace = true -deno_package_json.workspace = true -futures.workspace = true -lazy-regex.workspace = true -once_cell.workspace = true -path-clean = "=0.1.0" -regex.workspace = true -serde_json.workspace = true -thiserror.workspace = true -tokio.workspace = true -url.workspace = true diff --git a/ext/node_resolver/clippy.toml b/ext/node_resolver/clippy.toml deleted file mode 100644 index 86150781bb94f7..00000000000000 --- a/ext/node_resolver/clippy.toml +++ /dev/null @@ -1,48 +0,0 @@ -disallowed-methods = [ - { path = "std::env::current_dir", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::Path::canonicalize", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::Path::is_dir", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::Path::is_file", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::Path::is_symlink", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::Path::metadata", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::Path::read_dir", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::Path::read_link", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::Path::symlink_metadata", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::Path::try_exists", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::PathBuf::exists", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::PathBuf::canonicalize", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::PathBuf::is_dir", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::PathBuf::is_file", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::PathBuf::is_symlink", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::PathBuf::metadata", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::PathBuf::read_dir", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::PathBuf::read_link", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::PathBuf::symlink_metadata", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::PathBuf::try_exists", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::env::set_current_dir", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::env::temp_dir", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::canonicalize", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::copy", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::create_dir_all", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::create_dir", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::DirBuilder::new", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::hard_link", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::metadata", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::OpenOptions::new", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::read_dir", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::read_link", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::read_to_string", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::read", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::remove_dir_all", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::remove_dir", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::remove_file", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::rename", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::set_permissions", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::symlink_metadata", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::fs::write", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::Path::canonicalize", reason = "File system operations should be done using NodeResolverFs trait" }, - { path = "std::path::Path::exists", reason = "File system operations should be done using NodeResolverFs trait" }, -] -disallowed-types = [ - { path = "std::sync::Arc", reason = "use crate::sync::MaybeArc instead" }, -] diff --git a/ext/node_resolver/lib.rs b/ext/node_resolver/lib.rs deleted file mode 100644 index f03f7704869fe5..00000000000000 --- a/ext/node_resolver/lib.rs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -#![deny(clippy::print_stderr)] -#![deny(clippy::print_stdout)] - -pub mod analyze; -pub mod env; -pub mod errors; -mod npm; -mod package_json; -mod path; -mod resolution; -mod sync; - -pub use deno_package_json::PackageJson; -pub use npm::NpmResolver; -pub use npm::NpmResolverRc; -pub use package_json::load_pkg_json; -pub use package_json::PackageJsonThreadLocalCache; -pub use path::PathClean; -pub use resolution::parse_npm_pkg_name; -pub use resolution::NodeModuleKind; -pub use resolution::NodeResolution; -pub use resolution::NodeResolutionMode; -pub use resolution::NodeResolver; -pub use resolution::DEFAULT_CONDITIONS; -pub use resolution::REQUIRE_CONDITIONS; diff --git a/ext/node_resolver/npm.rs b/ext/node_resolver/npm.rs deleted file mode 100644 index 77df57c489d2e7..00000000000000 --- a/ext/node_resolver/npm.rs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -use std::path::Path; -use std::path::PathBuf; - -use url::Url; - -use crate::errors; -use crate::path::PathClean; -use crate::sync::MaybeSend; -use crate::sync::MaybeSync; - -#[allow(clippy::disallowed_types)] -pub type NpmResolverRc = crate::sync::MaybeArc; - -pub trait NpmResolver: std::fmt::Debug + MaybeSend + MaybeSync { - /// Resolves an npm package folder path from an npm package referrer. - fn resolve_package_folder_from_package( - &self, - specifier: &str, - referrer: &Url, - ) -> Result; - - fn in_npm_package(&self, specifier: &Url) -> bool; - - fn in_npm_package_at_dir_path(&self, path: &Path) -> bool { - let specifier = match Url::from_directory_path(path.to_path_buf().clean()) { - Ok(p) => p, - Err(_) => return false, - }; - self.in_npm_package(&specifier) - } - - fn in_npm_package_at_file_path(&self, path: &Path) -> bool { - let specifier = match Url::from_file_path(path.to_path_buf().clean()) { - Ok(p) => p, - Err(_) => return false, - }; - self.in_npm_package(&specifier) - } -} diff --git a/ext/node_resolver/package_json.rs b/ext/node_resolver/package_json.rs deleted file mode 100644 index de750f1d7e3db3..00000000000000 --- a/ext/node_resolver/package_json.rs +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -use deno_package_json::PackageJson; -use deno_package_json::PackageJsonRc; -use std::cell::RefCell; -use std::collections::HashMap; -use std::io::ErrorKind; -use std::path::Path; -use std::path::PathBuf; - -use crate::errors::PackageJsonLoadError; - -// use a thread local cache so that workers have their own distinct cache -thread_local! { - static CACHE: RefCell> = RefCell::new(HashMap::new()); -} - -pub struct PackageJsonThreadLocalCache; - -impl PackageJsonThreadLocalCache { - pub fn clear() { - CACHE.with(|cache| cache.borrow_mut().clear()); - } -} - -impl deno_package_json::PackageJsonCache for PackageJsonThreadLocalCache { - fn get(&self, path: &Path) -> Option { - CACHE.with(|cache| cache.borrow().get(path).cloned()) - } - - fn set(&self, path: PathBuf, package_json: PackageJsonRc) { - CACHE.with(|cache| cache.borrow_mut().insert(path, package_json)); - } -} - -/// Helper to load a package.json file using the thread local cache -/// in node_resolver. -pub fn load_pkg_json( - fs: &dyn deno_package_json::fs::DenoPkgJsonFs, - path: &Path, -) -> Result, PackageJsonLoadError> { - let result = - PackageJson::load_from_path(path, fs, Some(&PackageJsonThreadLocalCache)); - match result { - Ok(pkg_json) => Ok(Some(pkg_json)), - Err(deno_package_json::PackageJsonLoadError::Io { source, .. }) - if source.kind() == ErrorKind::NotFound => - { - Ok(None) - } - Err(err) => Err(PackageJsonLoadError(err)), - } -} diff --git a/ext/node_resolver/path.rs b/ext/node_resolver/path.rs deleted file mode 100644 index ece270cd9169ed..00000000000000 --- a/ext/node_resolver/path.rs +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -use std::path::Component; -use std::path::Path; -use std::path::PathBuf; - -use url::Url; - -/// Extension to path_clean::PathClean -pub trait PathClean { - fn clean(&self) -> T; -} - -impl PathClean for PathBuf { - fn clean(&self) -> PathBuf { - fn is_clean_path(path: &Path) -> bool { - let path = path.to_string_lossy(); - let mut current_index = 0; - while let Some(index) = path[current_index..].find("\\.") { - let trailing_index = index + current_index + 2; - let mut trailing_chars = path[trailing_index..].chars(); - match trailing_chars.next() { - Some('.') => match trailing_chars.next() { - Some('/') | Some('\\') | None => { - return false; - } - _ => {} - }, - Some('/') | Some('\\') => { - return false; - } - _ => {} - } - current_index = trailing_index; - } - true - } - - let path = path_clean::PathClean::clean(self); - if cfg!(windows) && !is_clean_path(&path) { - // temporary workaround because path_clean::PathClean::clean is - // not good enough on windows - let mut components = Vec::new(); - - for component in path.components() { - match component { - Component::CurDir => { - // skip - } - Component::ParentDir => { - let maybe_last_component = components.pop(); - if !matches!(maybe_last_component, Some(Component::Normal(_))) { - panic!("Error normalizing: {}", path.display()); - } - } - Component::Normal(_) | Component::RootDir | Component::Prefix(_) => { - components.push(component); - } - } - } - components.into_iter().collect::() - } else { - path - } - } -} - -pub(crate) fn to_file_specifier(path: &Path) -> Url { - match Url::from_file_path(path) { - Ok(url) => url, - Err(_) => panic!("Invalid path: {}", path.display()), - } -} - -// todo(dsherret): we have the below code also in deno_core and it -// would be good to somehow re-use it in both places (we don't want -// to create a dependency on deno_core here) - -#[cfg(not(windows))] -#[inline] -pub fn strip_unc_prefix(path: PathBuf) -> PathBuf { - path -} - -/// Strips the unc prefix (ex. \\?\) from Windows paths. -#[cfg(windows)] -pub fn strip_unc_prefix(path: PathBuf) -> PathBuf { - use std::path::Component; - use std::path::Prefix; - - let mut components = path.components(); - match components.next() { - Some(Component::Prefix(prefix)) => { - match prefix.kind() { - // \\?\device - Prefix::Verbatim(device) => { - let mut path = PathBuf::new(); - path.push(format!(r"\\{}\", device.to_string_lossy())); - path.extend(components.filter(|c| !matches!(c, Component::RootDir))); - path - } - // \\?\c:\path - Prefix::VerbatimDisk(_) => { - let mut path = PathBuf::new(); - path.push(prefix.as_os_str().to_string_lossy().replace(r"\\?\", "")); - path.extend(components); - path - } - // \\?\UNC\hostname\share_name\path - Prefix::VerbatimUNC(hostname, share_name) => { - let mut path = PathBuf::new(); - path.push(format!( - r"\\{}\{}\", - hostname.to_string_lossy(), - share_name.to_string_lossy() - )); - path.extend(components.filter(|c| !matches!(c, Component::RootDir))); - path - } - _ => path, - } - } - _ => path, - } -} - -#[cfg(test)] -mod test { - #[cfg(windows)] - #[test] - fn test_path_clean() { - use super::*; - - run_test("C:\\test\\./file.txt", "C:\\test\\file.txt"); - run_test("C:\\test\\../other/file.txt", "C:\\other\\file.txt"); - run_test("C:\\test\\../other\\file.txt", "C:\\other\\file.txt"); - - fn run_test(input: &str, expected: &str) { - assert_eq!(PathBuf::from(input).clean(), PathBuf::from(expected)); - } - } - - #[cfg(windows)] - #[test] - fn test_strip_unc_prefix() { - use std::path::PathBuf; - - run_test(r"C:\", r"C:\"); - run_test(r"C:\test\file.txt", r"C:\test\file.txt"); - - run_test(r"\\?\C:\", r"C:\"); - run_test(r"\\?\C:\test\file.txt", r"C:\test\file.txt"); - - run_test(r"\\.\C:\", r"\\.\C:\"); - run_test(r"\\.\C:\Test\file.txt", r"\\.\C:\Test\file.txt"); - - run_test(r"\\?\UNC\localhost\", r"\\localhost"); - run_test(r"\\?\UNC\localhost\c$\", r"\\localhost\c$"); - run_test( - r"\\?\UNC\localhost\c$\Windows\file.txt", - r"\\localhost\c$\Windows\file.txt", - ); - run_test(r"\\?\UNC\wsl$\deno.json", r"\\wsl$\deno.json"); - - run_test(r"\\?\server1", r"\\server1"); - run_test(r"\\?\server1\e$\", r"\\server1\e$\"); - run_test( - r"\\?\server1\e$\test\file.txt", - r"\\server1\e$\test\file.txt", - ); - - fn run_test(input: &str, expected: &str) { - assert_eq!( - super::strip_unc_prefix(PathBuf::from(input)), - PathBuf::from(expected) - ); - } - } -} diff --git a/ext/telemetry/Cargo.toml b/ext/telemetry/Cargo.toml new file mode 100644 index 00000000000000..12d7777491ffc0 --- /dev/null +++ b/ext/telemetry/Cargo.toml @@ -0,0 +1,31 @@ +# Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +[package] +name = "deno_telemetry" +version = "0.2.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +readme = "README.md" +repository.workspace = true +description = "Telemetry for Deno" + +[lib] +path = "lib.rs" + +[dependencies] +async-trait.workspace = true +deno_core.workspace = true +http-body-util.workspace = true +hyper.workspace = true +hyper-util.workspace = true +log.workspace = true +once_cell.workspace = true +opentelemetry.workspace = true +opentelemetry-http.workspace = true +opentelemetry-otlp.workspace = true +opentelemetry-semantic-conventions.workspace = true +opentelemetry_sdk.workspace = true +pin-project.workspace = true +serde.workspace = true +tokio.workspace = true diff --git a/ext/telemetry/README.md b/ext/telemetry/README.md new file mode 100644 index 00000000000000..6931a3b8964dc3 --- /dev/null +++ b/ext/telemetry/README.md @@ -0,0 +1,3 @@ +# `deno_telemetry` + +This crate implements telemetry for Deno using OpenTelemetry. diff --git a/ext/telemetry/lib.rs b/ext/telemetry/lib.rs new file mode 100644 index 00000000000000..1ce8ac1dcc77f5 --- /dev/null +++ b/ext/telemetry/lib.rs @@ -0,0 +1,874 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use deno_core::anyhow; +use deno_core::anyhow::anyhow; +use deno_core::futures::channel::mpsc; +use deno_core::futures::channel::mpsc::UnboundedSender; +use deno_core::futures::future::BoxFuture; +use deno_core::futures::stream; +use deno_core::futures::Stream; +use deno_core::futures::StreamExt; +use deno_core::op2; +use deno_core::v8; +use deno_core::OpState; +use once_cell::sync::Lazy; +use once_cell::sync::OnceCell; +use opentelemetry::logs::AnyValue; +use opentelemetry::logs::LogRecord as LogRecordTrait; +use opentelemetry::logs::Severity; +use opentelemetry::trace::SpanContext; +use opentelemetry::trace::SpanId; +use opentelemetry::trace::SpanKind; +use opentelemetry::trace::Status as SpanStatus; +use opentelemetry::trace::TraceFlags; +use opentelemetry::trace::TraceId; +use opentelemetry::Key; +use opentelemetry::KeyValue; +use opentelemetry::StringValue; +use opentelemetry::Value; +use opentelemetry_otlp::HttpExporterBuilder; +use opentelemetry_otlp::Protocol; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_otlp::WithHttpConfig; +use opentelemetry_sdk::export::trace::SpanData; +use opentelemetry_sdk::logs::BatchLogProcessor; +use opentelemetry_sdk::logs::LogProcessor as LogProcessorTrait; +use opentelemetry_sdk::logs::LogRecord; +use opentelemetry_sdk::trace::BatchSpanProcessor; +use opentelemetry_sdk::trace::SpanProcessor as SpanProcessorTrait; +use opentelemetry_sdk::Resource; +use opentelemetry_semantic_conventions::resource::PROCESS_RUNTIME_NAME; +use opentelemetry_semantic_conventions::resource::PROCESS_RUNTIME_VERSION; +use opentelemetry_semantic_conventions::resource::TELEMETRY_SDK_LANGUAGE; +use opentelemetry_semantic_conventions::resource::TELEMETRY_SDK_NAME; +use opentelemetry_semantic_conventions::resource::TELEMETRY_SDK_VERSION; +use serde::Deserialize; +use serde::Serialize; +use std::borrow::Cow; +use std::env; +use std::fmt::Debug; +use std::pin::Pin; +use std::task::Context; +use std::task::Poll; +use std::thread; +use std::time::Duration; +use std::time::SystemTime; + +type SpanProcessor = BatchSpanProcessor; +type LogProcessor = BatchLogProcessor; + +deno_core::extension!( + deno_telemetry, + ops = [ + op_otel_log, + op_otel_instrumentation_scope_create_and_enter, + op_otel_instrumentation_scope_enter, + op_otel_instrumentation_scope_enter_builtin, + op_otel_span_start, + op_otel_span_continue, + op_otel_span_attribute, + op_otel_span_attribute2, + op_otel_span_attribute3, + op_otel_span_set_dropped, + op_otel_span_flush, + ], + esm = ["telemetry.ts", "util.ts"], +); + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OtelConfig { + pub runtime_name: Cow<'static, str>, + pub runtime_version: Cow<'static, str>, + pub console: OtelConsoleConfig, + pub deterministic: bool, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[repr(u8)] +pub enum OtelConsoleConfig { + Ignore = 0, + Capture = 1, + Replace = 2, +} + +impl Default for OtelConfig { + fn default() -> Self { + Self { + runtime_name: Cow::Borrowed(env!("CARGO_PKG_NAME")), + runtime_version: Cow::Borrowed(env!("CARGO_PKG_VERSION")), + console: OtelConsoleConfig::Capture, + deterministic: false, + } + } +} + +static OTEL_SHARED_RUNTIME_SPAWN_TASK_TX: Lazy< + UnboundedSender>, +> = Lazy::new(otel_create_shared_runtime); + +fn otel_create_shared_runtime() -> UnboundedSender> { + let (spawn_task_tx, mut spawn_task_rx) = + mpsc::unbounded::>(); + + thread::spawn(move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_io() + .enable_time() + // This limits the number of threads for blocking operations (like for + // synchronous fs ops) or CPU bound tasks like when we run dprint in + // parallel for deno fmt. + // The default value is 512, which is an unhelpfully large thread pool. We + // don't ever want to have more than a couple dozen threads. + .max_blocking_threads(if cfg!(windows) { + // on windows, tokio uses blocking tasks for child process IO, make sure + // we have enough available threads for other tasks to run + 4 * std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(8) + } else { + 32 + }) + .build() + .unwrap(); + + rt.block_on(async move { + while let Some(task) = spawn_task_rx.next().await { + tokio::spawn(task); + } + }); + }); + + spawn_task_tx +} + +#[derive(Clone, Copy)] +struct OtelSharedRuntime; + +impl hyper::rt::Executor> for OtelSharedRuntime { + fn execute(&self, fut: BoxFuture<'static, ()>) { + (*OTEL_SHARED_RUNTIME_SPAWN_TASK_TX) + .unbounded_send(fut) + .expect("failed to send task to shared OpenTelemetry runtime"); + } +} + +impl opentelemetry_sdk::runtime::Runtime for OtelSharedRuntime { + type Interval = Pin + Send + 'static>>; + type Delay = Pin>; + + fn interval(&self, period: Duration) -> Self::Interval { + stream::repeat(()) + .then(move |_| tokio::time::sleep(period)) + .boxed() + } + + fn spawn(&self, future: BoxFuture<'static, ()>) { + (*OTEL_SHARED_RUNTIME_SPAWN_TASK_TX) + .unbounded_send(future) + .expect("failed to send task to shared OpenTelemetry runtime"); + } + + fn delay(&self, duration: Duration) -> Self::Delay { + Box::pin(tokio::time::sleep(duration)) + } +} + +impl opentelemetry_sdk::runtime::RuntimeChannel for OtelSharedRuntime { + type Receiver = BatchMessageChannelReceiver; + type Sender = BatchMessageChannelSender; + + fn batch_message_channel( + &self, + capacity: usize, + ) -> (Self::Sender, Self::Receiver) { + let (batch_tx, batch_rx) = tokio::sync::mpsc::channel::(capacity); + (batch_tx.into(), batch_rx.into()) + } +} + +#[derive(Debug)] +pub struct BatchMessageChannelSender { + sender: tokio::sync::mpsc::Sender, +} + +impl From> + for BatchMessageChannelSender +{ + fn from(sender: tokio::sync::mpsc::Sender) -> Self { + Self { sender } + } +} + +impl opentelemetry_sdk::runtime::TrySend + for BatchMessageChannelSender +{ + type Message = T; + + fn try_send( + &self, + item: Self::Message, + ) -> Result<(), opentelemetry_sdk::runtime::TrySendError> { + self.sender.try_send(item).map_err(|err| match err { + tokio::sync::mpsc::error::TrySendError::Full(_) => { + opentelemetry_sdk::runtime::TrySendError::ChannelFull + } + tokio::sync::mpsc::error::TrySendError::Closed(_) => { + opentelemetry_sdk::runtime::TrySendError::ChannelClosed + } + }) + } +} + +pub struct BatchMessageChannelReceiver { + receiver: tokio::sync::mpsc::Receiver, +} + +impl From> + for BatchMessageChannelReceiver +{ + fn from(receiver: tokio::sync::mpsc::Receiver) -> Self { + Self { receiver } + } +} + +impl Stream for BatchMessageChannelReceiver { + type Item = T; + + fn poll_next( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + self.receiver.poll_recv(cx) + } +} + +mod hyper_client { + use http_body_util::BodyExt; + use http_body_util::Full; + use hyper::body::Body as HttpBody; + use hyper::body::Frame; + use hyper_util::client::legacy::connect::HttpConnector; + use hyper_util::client::legacy::Client; + use opentelemetry_http::Bytes; + use opentelemetry_http::HttpError; + use opentelemetry_http::Request; + use opentelemetry_http::Response; + use opentelemetry_http::ResponseExt; + use std::fmt::Debug; + use std::pin::Pin; + use std::task::Poll; + use std::task::{self}; + + use super::OtelSharedRuntime; + + // same as opentelemetry_http::HyperClient except it uses OtelSharedRuntime + #[derive(Debug, Clone)] + pub struct HyperClient { + inner: Client, + } + + impl HyperClient { + pub fn new() -> Self { + Self { + inner: Client::builder(OtelSharedRuntime).build(HttpConnector::new()), + } + } + } + + #[async_trait::async_trait] + impl opentelemetry_http::HttpClient for HyperClient { + async fn send( + &self, + request: Request>, + ) -> Result, HttpError> { + let (parts, body) = request.into_parts(); + let request = Request::from_parts(parts, Body(Full::from(body))); + let mut response = self.inner.request(request).await?; + let headers = std::mem::take(response.headers_mut()); + + let mut http_response = Response::builder() + .status(response.status()) + .body(response.into_body().collect().await?.to_bytes())?; + *http_response.headers_mut() = headers; + + Ok(http_response.error_for_status()?) + } + } + + #[pin_project::pin_project] + pub struct Body(#[pin] Full); + + impl HttpBody for Body { + type Data = Bytes; + type Error = Box; + + #[inline] + fn poll_frame( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + ) -> Poll, Self::Error>>> { + self.project().0.poll_frame(cx).map_err(Into::into) + } + + #[inline] + fn is_end_stream(&self) -> bool { + self.0.is_end_stream() + } + + #[inline] + fn size_hint(&self) -> hyper::body::SizeHint { + self.0.size_hint() + } + } +} + +static OTEL_PROCESSORS: OnceCell<(SpanProcessor, LogProcessor)> = + OnceCell::new(); + +static BUILT_IN_INSTRUMENTATION_SCOPE: OnceCell< + opentelemetry::InstrumentationScope, +> = OnceCell::new(); + +pub fn init(config: OtelConfig) -> anyhow::Result<()> { + // Parse the `OTEL_EXPORTER_OTLP_PROTOCOL` variable. The opentelemetry_* + // crates don't do this automatically. + // TODO(piscisaureus): enable GRPC support. + let protocol = match env::var("OTEL_EXPORTER_OTLP_PROTOCOL").as_deref() { + Ok("http/protobuf") => Protocol::HttpBinary, + Ok("http/json") => Protocol::HttpJson, + Ok("") | Err(env::VarError::NotPresent) => { + return Ok(()); + } + Ok(protocol) => { + return Err(anyhow!( + "Env var OTEL_EXPORTER_OTLP_PROTOCOL specifies an unsupported protocol: {}", + protocol + )); + } + Err(err) => { + return Err(anyhow!( + "Failed to read env var OTEL_EXPORTER_OTLP_PROTOCOL: {}", + err + )); + } + }; + + // Define the resource attributes that will be attached to all log records. + // These attributes are sourced as follows (in order of precedence): + // * The `service.name` attribute from the `OTEL_SERVICE_NAME` env var. + // * Additional attributes from the `OTEL_RESOURCE_ATTRIBUTES` env var. + // * Default attribute values defined here. + // TODO(piscisaureus): add more default attributes (e.g. script path). + let mut resource = Resource::default(); + + // Add the runtime name and version to the resource attributes. Also override + // the `telemetry.sdk` attributes to include the Deno runtime. + resource = resource.merge(&Resource::new(vec![ + KeyValue::new(PROCESS_RUNTIME_NAME, config.runtime_name), + KeyValue::new(PROCESS_RUNTIME_VERSION, config.runtime_version.clone()), + KeyValue::new( + TELEMETRY_SDK_LANGUAGE, + format!( + "deno-{}", + resource.get(Key::new(TELEMETRY_SDK_LANGUAGE)).unwrap() + ), + ), + KeyValue::new( + TELEMETRY_SDK_NAME, + format!( + "deno-{}", + resource.get(Key::new(TELEMETRY_SDK_NAME)).unwrap() + ), + ), + KeyValue::new( + TELEMETRY_SDK_VERSION, + format!( + "{}-{}", + config.runtime_version, + resource.get(Key::new(TELEMETRY_SDK_VERSION)).unwrap() + ), + ), + ])); + + // The OTLP endpoint is automatically picked up from the + // `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable. Additional headers can + // be specified using `OTEL_EXPORTER_OTLP_HEADERS`. + + let client = hyper_client::HyperClient::new(); + + let span_exporter = HttpExporterBuilder::default() + .with_http_client(client.clone()) + .with_protocol(protocol) + .build_span_exporter()?; + let mut span_processor = + BatchSpanProcessor::builder(span_exporter, OtelSharedRuntime).build(); + span_processor.set_resource(&resource); + + let log_exporter = HttpExporterBuilder::default() + .with_http_client(client) + .with_protocol(protocol) + .build_log_exporter()?; + let log_processor = + BatchLogProcessor::builder(log_exporter, OtelSharedRuntime).build(); + log_processor.set_resource(&resource); + + OTEL_PROCESSORS + .set((span_processor, log_processor)) + .map_err(|_| anyhow!("failed to init otel"))?; + + let builtin_instrumentation_scope = + opentelemetry::InstrumentationScope::builder("deno") + .with_version(config.runtime_version.clone()) + .build(); + BUILT_IN_INSTRUMENTATION_SCOPE + .set(builtin_instrumentation_scope) + .map_err(|_| anyhow!("failed to init otel"))?; + + Ok(()) +} + +/// This function is called by the runtime whenever it is about to call +/// `process::exit()`, to ensure that all OpenTelemetry logs are properly +/// flushed before the process terminates. +pub fn flush() { + if let Some((span_processor, log_processor)) = OTEL_PROCESSORS.get() { + let _ = span_processor.force_flush(); + let _ = log_processor.force_flush(); + } +} + +pub fn handle_log(record: &log::Record) { + use log::Level; + + let Some((_, log_processor)) = OTEL_PROCESSORS.get() else { + return; + }; + + let mut log_record = LogRecord::default(); + + log_record.set_observed_timestamp(SystemTime::now()); + log_record.set_severity_number(match record.level() { + Level::Error => Severity::Error, + Level::Warn => Severity::Warn, + Level::Info => Severity::Info, + Level::Debug => Severity::Debug, + Level::Trace => Severity::Trace, + }); + log_record.set_severity_text(record.level().as_str()); + log_record.set_body(record.args().to_string().into()); + log_record.set_target(record.metadata().target().to_string()); + + struct Visitor<'s>(&'s mut LogRecord); + + impl<'s, 'kvs> log::kv::VisitSource<'kvs> for Visitor<'s> { + fn visit_pair( + &mut self, + key: log::kv::Key<'kvs>, + value: log::kv::Value<'kvs>, + ) -> Result<(), log::kv::Error> { + #[allow(clippy::manual_map)] + let value = if let Some(v) = value.to_bool() { + Some(AnyValue::Boolean(v)) + } else if let Some(v) = value.to_borrowed_str() { + Some(AnyValue::String(v.to_owned().into())) + } else if let Some(v) = value.to_f64() { + Some(AnyValue::Double(v)) + } else if let Some(v) = value.to_i64() { + Some(AnyValue::Int(v)) + } else { + None + }; + + if let Some(value) = value { + let key = Key::from(key.as_str().to_owned()); + self.0.add_attribute(key, value); + } + + Ok(()) + } + } + + let _ = record.key_values().visit(&mut Visitor(&mut log_record)); + + log_processor.emit( + &mut log_record, + BUILT_IN_INSTRUMENTATION_SCOPE.get().unwrap(), + ); +} + +fn parse_trace_id( + scope: &mut v8::HandleScope<'_>, + trace_id: v8::Local<'_, v8::Value>, +) -> TraceId { + if let Ok(string) = trace_id.try_cast() { + let value_view = v8::ValueView::new(scope, string); + match value_view.data() { + v8::ValueViewData::OneByte(bytes) => { + TraceId::from_hex(&String::from_utf8_lossy(bytes)) + .unwrap_or(TraceId::INVALID) + } + + _ => TraceId::INVALID, + } + } else if let Ok(uint8array) = trace_id.try_cast::() { + let data = uint8array.data(); + let byte_length = uint8array.byte_length(); + if byte_length != 16 { + return TraceId::INVALID; + } + // SAFETY: We have ensured that the byte length is 16, so it is safe to + // cast the data to an array of 16 bytes. + let bytes = unsafe { &*(data as *const u8 as *const [u8; 16]) }; + TraceId::from_bytes(*bytes) + } else { + TraceId::INVALID + } +} + +fn parse_span_id( + scope: &mut v8::HandleScope<'_>, + span_id: v8::Local<'_, v8::Value>, +) -> SpanId { + if let Ok(string) = span_id.try_cast() { + let value_view = v8::ValueView::new(scope, string); + match value_view.data() { + v8::ValueViewData::OneByte(bytes) => { + SpanId::from_hex(&String::from_utf8_lossy(bytes)) + .unwrap_or(SpanId::INVALID) + } + _ => SpanId::INVALID, + } + } else if let Ok(uint8array) = span_id.try_cast::() { + let data = uint8array.data(); + let byte_length = uint8array.byte_length(); + if byte_length != 8 { + return SpanId::INVALID; + } + // SAFETY: We have ensured that the byte length is 8, so it is safe to + // cast the data to an array of 8 bytes. + let bytes = unsafe { &*(data as *const u8 as *const [u8; 8]) }; + SpanId::from_bytes(*bytes) + } else { + SpanId::INVALID + } +} + +macro_rules! attr { + ($scope:ident, $attributes:expr $(=> $dropped_attributes_count:expr)?, $name:expr, $value:expr) => { + let name = if let Ok(name) = $name.try_cast() { + let view = v8::ValueView::new($scope, name); + match view.data() { + v8::ValueViewData::OneByte(bytes) => { + Some(String::from_utf8_lossy(bytes).into_owned()) + } + v8::ValueViewData::TwoByte(bytes) => { + Some(String::from_utf16_lossy(bytes)) + } + } + } else { + None + }; + let value = if let Ok(string) = $value.try_cast::() { + Some(Value::String(StringValue::from({ + let x = v8::ValueView::new($scope, string); + match x.data() { + v8::ValueViewData::OneByte(bytes) => { + String::from_utf8_lossy(bytes).into_owned() + } + v8::ValueViewData::TwoByte(bytes) => String::from_utf16_lossy(bytes), + } + }))) + } else if let Ok(number) = $value.try_cast::() { + Some(Value::F64(number.value())) + } else if let Ok(boolean) = $value.try_cast::() { + Some(Value::Bool(boolean.is_true())) + } else if let Ok(bigint) = $value.try_cast::() { + let (i64_value, _lossless) = bigint.i64_value(); + Some(Value::I64(i64_value)) + } else { + None + }; + if let (Some(name), Some(value)) = (name, value) { + $attributes.push(KeyValue::new(name, value)); + } + $( + else { + $dropped_attributes_count += 1; + } + )? + }; +} + +#[derive(Debug, Clone)] +struct InstrumentationScope(opentelemetry::InstrumentationScope); + +impl deno_core::GarbageCollected for InstrumentationScope {} + +#[op2] +#[cppgc] +fn op_otel_instrumentation_scope_create_and_enter( + state: &mut OpState, + #[string] name: String, + #[string] version: Option, + #[string] schema_url: Option, +) -> InstrumentationScope { + let mut builder = opentelemetry::InstrumentationScope::builder(name); + if let Some(version) = version { + builder = builder.with_version(version); + } + if let Some(schema_url) = schema_url { + builder = builder.with_schema_url(schema_url); + } + let scope = InstrumentationScope(builder.build()); + state.put(scope.clone()); + scope +} + +#[op2(fast)] +fn op_otel_instrumentation_scope_enter( + state: &mut OpState, + #[cppgc] scope: &InstrumentationScope, +) { + state.put(scope.clone()); +} + +#[op2(fast)] +fn op_otel_instrumentation_scope_enter_builtin(state: &mut OpState) { + state.put(InstrumentationScope( + BUILT_IN_INSTRUMENTATION_SCOPE.get().unwrap().clone(), + )); +} + +#[op2(fast)] +fn op_otel_log( + scope: &mut v8::HandleScope<'_>, + #[string] message: String, + #[smi] level: i32, + trace_id: v8::Local<'_, v8::Value>, + span_id: v8::Local<'_, v8::Value>, + #[smi] trace_flags: u8, +) { + let Some((_, log_processor)) = OTEL_PROCESSORS.get() else { + return; + }; + + // Convert the integer log level that ext/console uses to the corresponding + // OpenTelemetry log severity. + let severity = match level { + ..=0 => Severity::Debug, + 1 => Severity::Info, + 2 => Severity::Warn, + 3.. => Severity::Error, + }; + + let trace_id = parse_trace_id(scope, trace_id); + let span_id = parse_span_id(scope, span_id); + + let mut log_record = LogRecord::default(); + + log_record.set_observed_timestamp(SystemTime::now()); + log_record.set_body(message.into()); + log_record.set_severity_number(severity); + log_record.set_severity_text(severity.name()); + if trace_id != TraceId::INVALID && span_id != SpanId::INVALID { + log_record.set_trace_context( + trace_id, + span_id, + Some(TraceFlags::new(trace_flags)), + ); + } + + log_processor.emit( + &mut log_record, + BUILT_IN_INSTRUMENTATION_SCOPE.get().unwrap(), + ); +} + +struct TemporarySpan(SpanData); + +#[allow(clippy::too_many_arguments)] +#[op2(fast)] +fn op_otel_span_start<'s>( + scope: &mut v8::HandleScope<'s>, + state: &mut OpState, + trace_id: v8::Local<'s, v8::Value>, + span_id: v8::Local<'s, v8::Value>, + parent_span_id: v8::Local<'s, v8::Value>, + #[smi] span_kind: u8, + name: v8::Local<'s, v8::Value>, + start_time: f64, + end_time: f64, +) -> Result<(), anyhow::Error> { + if let Some(temporary_span) = state.try_take::() { + let Some((span_processor, _)) = OTEL_PROCESSORS.get() else { + return Ok(()); + }; + span_processor.on_end(temporary_span.0); + }; + + let Some(InstrumentationScope(instrumentation_scope)) = + state.try_borrow::() + else { + return Err(anyhow!("instrumentation scope not available")); + }; + + let trace_id = parse_trace_id(scope, trace_id); + if trace_id == TraceId::INVALID { + return Err(anyhow!("invalid trace_id")); + } + + let span_id = parse_span_id(scope, span_id); + if span_id == SpanId::INVALID { + return Err(anyhow!("invalid span_id")); + } + + let parent_span_id = parse_span_id(scope, parent_span_id); + + let name = { + let x = v8::ValueView::new(scope, name.try_cast()?); + match x.data() { + v8::ValueViewData::OneByte(bytes) => { + String::from_utf8_lossy(bytes).into_owned() + } + v8::ValueViewData::TwoByte(bytes) => String::from_utf16_lossy(bytes), + } + }; + + let temporary_span = TemporarySpan(SpanData { + span_context: SpanContext::new( + trace_id, + span_id, + TraceFlags::SAMPLED, + false, + Default::default(), + ), + parent_span_id, + span_kind: match span_kind { + 0 => SpanKind::Internal, + 1 => SpanKind::Server, + 2 => SpanKind::Client, + 3 => SpanKind::Producer, + 4 => SpanKind::Consumer, + _ => return Err(anyhow!("invalid span kind")), + }, + name: Cow::Owned(name), + start_time: SystemTime::UNIX_EPOCH + .checked_add(std::time::Duration::from_secs_f64(start_time)) + .ok_or_else(|| anyhow!("invalid start time"))?, + end_time: SystemTime::UNIX_EPOCH + .checked_add(std::time::Duration::from_secs_f64(end_time)) + .ok_or_else(|| anyhow!("invalid start time"))?, + attributes: Vec::new(), + dropped_attributes_count: 0, + events: Default::default(), + links: Default::default(), + status: SpanStatus::Unset, + instrumentation_scope: instrumentation_scope.clone(), + }); + state.put(temporary_span); + + Ok(()) +} + +#[op2(fast)] +fn op_otel_span_continue( + state: &mut OpState, + #[smi] status: u8, + #[string] error_description: Cow<'_, str>, +) { + if let Some(temporary_span) = state.try_borrow_mut::() { + temporary_span.0.status = match status { + 0 => SpanStatus::Unset, + 1 => SpanStatus::Ok, + 2 => SpanStatus::Error { + description: Cow::Owned(error_description.into_owned()), + }, + _ => return, + }; + } +} + +#[op2(fast)] +fn op_otel_span_attribute<'s>( + scope: &mut v8::HandleScope<'s>, + state: &mut OpState, + #[smi] capacity: u32, + key: v8::Local<'s, v8::Value>, + value: v8::Local<'s, v8::Value>, +) { + if let Some(temporary_span) = state.try_borrow_mut::() { + temporary_span.0.attributes.reserve_exact( + (capacity as usize) - temporary_span.0.attributes.capacity(), + ); + attr!(scope, temporary_span.0.attributes => temporary_span.0.dropped_attributes_count, key, value); + } +} + +#[op2(fast)] +fn op_otel_span_attribute2<'s>( + scope: &mut v8::HandleScope<'s>, + state: &mut OpState, + #[smi] capacity: u32, + key1: v8::Local<'s, v8::Value>, + value1: v8::Local<'s, v8::Value>, + key2: v8::Local<'s, v8::Value>, + value2: v8::Local<'s, v8::Value>, +) { + if let Some(temporary_span) = state.try_borrow_mut::() { + temporary_span.0.attributes.reserve_exact( + (capacity as usize) - temporary_span.0.attributes.capacity(), + ); + attr!(scope, temporary_span.0.attributes => temporary_span.0.dropped_attributes_count, key1, value1); + attr!(scope, temporary_span.0.attributes => temporary_span.0.dropped_attributes_count, key2, value2); + } +} + +#[allow(clippy::too_many_arguments)] +#[op2(fast)] +fn op_otel_span_attribute3<'s>( + scope: &mut v8::HandleScope<'s>, + state: &mut OpState, + #[smi] capacity: u32, + key1: v8::Local<'s, v8::Value>, + value1: v8::Local<'s, v8::Value>, + key2: v8::Local<'s, v8::Value>, + value2: v8::Local<'s, v8::Value>, + key3: v8::Local<'s, v8::Value>, + value3: v8::Local<'s, v8::Value>, +) { + if let Some(temporary_span) = state.try_borrow_mut::() { + temporary_span.0.attributes.reserve_exact( + (capacity as usize) - temporary_span.0.attributes.capacity(), + ); + attr!(scope, temporary_span.0.attributes => temporary_span.0.dropped_attributes_count, key1, value1); + attr!(scope, temporary_span.0.attributes => temporary_span.0.dropped_attributes_count, key2, value2); + attr!(scope, temporary_span.0.attributes => temporary_span.0.dropped_attributes_count, key3, value3); + } +} + +#[op2(fast)] +fn op_otel_span_set_dropped( + state: &mut OpState, + #[smi] dropped_attributes_count: u32, + #[smi] dropped_links_count: u32, + #[smi] dropped_events_count: u32, +) { + if let Some(temporary_span) = state.try_borrow_mut::() { + temporary_span.0.dropped_attributes_count += dropped_attributes_count; + temporary_span.0.links.dropped_count += dropped_links_count; + temporary_span.0.events.dropped_count += dropped_events_count; + } +} + +#[op2(fast)] +fn op_otel_span_flush(state: &mut OpState) { + let Some(temporary_span) = state.try_take::() else { + return; + }; + + let Some((span_processor, _)) = OTEL_PROCESSORS.get() else { + return; + }; + + span_processor.on_end(temporary_span.0); +} diff --git a/ext/telemetry/telemetry.ts b/ext/telemetry/telemetry.ts new file mode 100644 index 00000000000000..03fbd83e2f6599 --- /dev/null +++ b/ext/telemetry/telemetry.ts @@ -0,0 +1,711 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +import { core, primordials } from "ext:core/mod.js"; +import { + op_crypto_get_random_values, + op_otel_instrumentation_scope_create_and_enter, + op_otel_instrumentation_scope_enter, + op_otel_instrumentation_scope_enter_builtin, + op_otel_log, + op_otel_span_attribute, + op_otel_span_attribute2, + op_otel_span_attribute3, + op_otel_span_continue, + op_otel_span_flush, + op_otel_span_set_dropped, + op_otel_span_start, +} from "ext:core/ops"; +import { Console } from "ext:deno_console/01_console.js"; +import { performance } from "ext:deno_web/15_performance.js"; + +const { + SafeWeakMap, + Array, + ObjectEntries, + SafeMap, + ReflectApply, + SymbolFor, + Error, + Uint8Array, + TypedArrayPrototypeSubarray, + ObjectAssign, + ObjectDefineProperty, + WeakRefPrototypeDeref, + String, + StringPrototypePadStart, + ObjectPrototypeIsPrototypeOf, + SafeWeakRef, +} = primordials; +const { AsyncVariable, setAsyncContext } = core; + +export let TRACING_ENABLED = false; +let DETERMINISTIC = false; + +// Note: These start at 0 in the JS library, +// but start at 1 when serialized with JSON. +enum SpanKind { + INTERNAL = 0, + SERVER = 1, + CLIENT = 2, + PRODUCER = 3, + CONSUMER = 4, +} + +interface TraceState { + set(key: string, value: string): TraceState; + unset(key: string): TraceState; + get(key: string): string | undefined; + serialize(): string; +} + +interface SpanContext { + traceId: string; + spanId: string; + isRemote?: boolean; + traceFlags: number; + traceState?: TraceState; +} + +type HrTime = [number, number]; + +enum SpanStatusCode { + UNSET = 0, + OK = 1, + ERROR = 2, +} + +interface SpanStatus { + code: SpanStatusCode; + message?: string; +} + +export type AttributeValue = + | string + | number + | boolean + | Array + | Array + | Array; + +interface Attributes { + [attributeKey: string]: AttributeValue | undefined; +} + +type SpanAttributes = Attributes; + +interface SpanOptions { + attributes?: Attributes; + kind?: SpanKind; +} + +interface Link { + context: SpanContext; + attributes?: SpanAttributes; + droppedAttributesCount?: number; +} + +interface TimedEvent { + time: HrTime; + name: string; + attributes?: SpanAttributes; + droppedAttributesCount?: number; +} + +interface IArrayValue { + values: IAnyValue[]; +} + +interface IAnyValue { + stringValue?: string | null; + boolValue?: boolean | null; + intValue?: number | null; + doubleValue?: number | null; + arrayValue?: IArrayValue; + kvlistValue?: IKeyValueList; + bytesValue?: Uint8Array; +} + +interface IKeyValueList { + values: IKeyValue[]; +} + +interface IKeyValue { + key: string; + value: IAnyValue; +} +interface IResource { + attributes: IKeyValue[]; + droppedAttributesCount: number; +} + +interface InstrumentationLibrary { + readonly name: string; + readonly version?: string; + readonly schemaUrl?: string; +} + +interface ReadableSpan { + readonly name: string; + readonly kind: SpanKind; + readonly spanContext: () => SpanContext; + readonly parentSpanId?: string; + readonly startTime: HrTime; + readonly endTime: HrTime; + readonly status: SpanStatus; + readonly attributes: SpanAttributes; + readonly links: Link[]; + readonly events: TimedEvent[]; + readonly duration: HrTime; + readonly ended: boolean; + readonly resource: IResource; + readonly instrumentationLibrary: InstrumentationLibrary; + readonly droppedAttributesCount: number; + readonly droppedEventsCount: number; + readonly droppedLinksCount: number; +} + +enum ExportResultCode { + SUCCESS = 0, + FAILED = 1, +} + +interface ExportResult { + code: ExportResultCode; + error?: Error; +} + +function hrToSecs(hr: [number, number]): number { + return ((hr[0] * 1e3 + hr[1] / 1e6) / 1000); +} + +const TRACE_FLAG_SAMPLED = 1 << 0; + +const instrumentationScopes = new SafeWeakMap< + InstrumentationLibrary, + { __key: "instrumentation-library" } +>(); +let activeInstrumentationLibrary: WeakRef | null = null; + +function submit( + spanId: string | Uint8Array, + traceId: string | Uint8Array, + traceFlags: number, + parentSpanId: string | Uint8Array | null, + span: Omit< + ReadableSpan, + | "spanContext" + | "startTime" + | "endTime" + | "parentSpanId" + | "duration" + | "ended" + | "resource" + >, + startTime: number, + endTime: number, +) { + if (!(traceFlags & TRACE_FLAG_SAMPLED)) return; + + // TODO(@lucacasonato): `resource` is ignored for now, should we implement it? + + const instrumentationLibrary = span.instrumentationLibrary; + if ( + !activeInstrumentationLibrary || + WeakRefPrototypeDeref(activeInstrumentationLibrary) !== + instrumentationLibrary + ) { + activeInstrumentationLibrary = new SafeWeakRef(instrumentationLibrary); + if (instrumentationLibrary === BUILTIN_INSTRUMENTATION_LIBRARY) { + op_otel_instrumentation_scope_enter_builtin(); + } else { + let instrumentationScope = instrumentationScopes + .get(instrumentationLibrary); + + if (instrumentationScope === undefined) { + instrumentationScope = op_otel_instrumentation_scope_create_and_enter( + instrumentationLibrary.name, + instrumentationLibrary.version, + instrumentationLibrary.schemaUrl, + ) as { __key: "instrumentation-library" }; + instrumentationScopes.set( + instrumentationLibrary, + instrumentationScope, + ); + } else { + op_otel_instrumentation_scope_enter( + instrumentationScope, + ); + } + } + } + + op_otel_span_start( + traceId, + spanId, + parentSpanId, + span.kind, + span.name, + startTime, + endTime, + ); + + const status = span.status; + if (status !== null && status.code !== 0) { + op_otel_span_continue(status.code, status.message ?? ""); + } + + const attributeKvs = ObjectEntries(span.attributes); + let i = 0; + while (i < attributeKvs.length) { + if (i + 2 < attributeKvs.length) { + op_otel_span_attribute3( + attributeKvs.length, + attributeKvs[i][0], + attributeKvs[i][1], + attributeKvs[i + 1][0], + attributeKvs[i + 1][1], + attributeKvs[i + 2][0], + attributeKvs[i + 2][1], + ); + i += 3; + } else if (i + 1 < attributeKvs.length) { + op_otel_span_attribute2( + attributeKvs.length, + attributeKvs[i][0], + attributeKvs[i][1], + attributeKvs[i + 1][0], + attributeKvs[i + 1][1], + ); + i += 2; + } else { + op_otel_span_attribute( + attributeKvs.length, + attributeKvs[i][0], + attributeKvs[i][1], + ); + i += 1; + } + } + + // TODO(@lucacasonato): implement links + // TODO(@lucacasonato): implement events + + const droppedAttributesCount = span.droppedAttributesCount; + const droppedLinksCount = span.droppedLinksCount + span.links.length; + const droppedEventsCount = span.droppedEventsCount + span.events.length; + if ( + droppedAttributesCount > 0 || droppedLinksCount > 0 || + droppedEventsCount > 0 + ) { + op_otel_span_set_dropped( + droppedAttributesCount, + droppedLinksCount, + droppedEventsCount, + ); + } + + op_otel_span_flush(); +} + +const now = () => (performance.timeOrigin + performance.now()) / 1000; + +const SPAN_ID_BYTES = 8; +const TRACE_ID_BYTES = 16; + +const INVALID_TRACE_ID = new Uint8Array(TRACE_ID_BYTES); +const INVALID_SPAN_ID = new Uint8Array(SPAN_ID_BYTES); + +const NO_ASYNC_CONTEXT = {}; + +let otelLog: (message: string, level: number) => void; + +const hexSliceLookupTable = (function () { + const alphabet = "0123456789abcdef"; + const table = new Array(256); + for (let i = 0; i < 16; ++i) { + const i16 = i * 16; + for (let j = 0; j < 16; ++j) { + table[i16 + j] = alphabet[i] + alphabet[j]; + } + } + return table; +})(); + +function bytesToHex(bytes: Uint8Array): string { + let out = ""; + for (let i = 0; i < bytes.length; i += 1) { + out += hexSliceLookupTable[bytes[i]]; + } + return out; +} + +const SPAN_KEY = SymbolFor("OpenTelemetry Context Key SPAN"); + +const BUILTIN_INSTRUMENTATION_LIBRARY: InstrumentationLibrary = {} as never; + +let COUNTER = 1; + +export let enterSpan: (span: Span) => void; +export let exitSpan: (span: Span) => void; +export let endSpan: (span: Span) => void; + +export class Span { + #traceId: string | Uint8Array; + #spanId: Uint8Array; + #traceFlags = TRACE_FLAG_SAMPLED; + + #spanContext: SpanContext | null = null; + + #parentSpanId: string | Uint8Array | null = null; + #parentSpanIdString: string | null = null; + + #recording = TRACING_ENABLED; + + #kind: number = SpanKind.INTERNAL; + #name: string; + #startTime: number; + #status: { code: number; message?: string } | null = null; + #attributes: Attributes = { __proto__: null } as never; + + #droppedEventsCount = 0; + #droppedLinksCount = 0; + + #asyncContext = NO_ASYNC_CONTEXT; + + static { + otelLog = function otelLog(message, level) { + let traceId = null; + let spanId = null; + let traceFlags = 0; + const span = CURRENT.get()?.getValue(SPAN_KEY); + if (span) { + // The lint is wrong, we can not use anything but `in` here because this + // is a private field. + // deno-lint-ignore prefer-primordials + if (#traceId in span) { + traceId = span.#traceId; + spanId = span.#spanId; + traceFlags = span.#traceFlags; + } else { + const context = span.spanContext(); + traceId = context.traceId; + spanId = context.spanId; + traceFlags = context.traceFlags; + } + } + return op_otel_log(message, level, traceId, spanId, traceFlags); + }; + + enterSpan = (span: Span) => { + if (!span.#recording) return; + const context = (CURRENT.get() || ROOT_CONTEXT).setValue(SPAN_KEY, span); + span.#asyncContext = CURRENT.enter(context); + }; + + exitSpan = (span: Span) => { + if (!span.#recording) return; + if (span.#asyncContext === NO_ASYNC_CONTEXT) return; + setAsyncContext(span.#asyncContext); + span.#asyncContext = NO_ASYNC_CONTEXT; + }; + + endSpan = (span: Span) => { + const endTime = now(); + submit( + span.#spanId, + span.#traceId, + span.#traceFlags, + span.#parentSpanId, + { + name: span.#name, + kind: span.#kind, + status: span.#status ?? { code: 0 }, + attributes: span.#attributes, + events: [], + links: [], + droppedAttributesCount: 0, + droppedEventsCount: span.#droppedEventsCount, + droppedLinksCount: span.#droppedLinksCount, + instrumentationLibrary: BUILTIN_INSTRUMENTATION_LIBRARY, + }, + span.#startTime, + endTime, + ); + }; + } + + constructor( + name: string, + options?: SpanOptions, + ) { + if (!this.isRecording) { + this.#name = ""; + this.#startTime = 0; + this.#traceId = INVALID_TRACE_ID; + this.#spanId = INVALID_SPAN_ID; + this.#traceFlags = 0; + return; + } + + this.#name = name; + this.#startTime = now(); + this.#attributes = options?.attributes ?? { __proto__: null } as never; + this.#kind = options?.kind ?? SpanKind.INTERNAL; + + const currentSpan: Span | { + spanContext(): { traceId: string; spanId: string }; + } = CURRENT.get()?.getValue(SPAN_KEY); + if (currentSpan) { + if (DETERMINISTIC) { + this.#spanId = StringPrototypePadStart(String(COUNTER++), 16, "0"); + } else { + this.#spanId = new Uint8Array(SPAN_ID_BYTES); + op_crypto_get_random_values(this.#spanId); + } + // deno-lint-ignore prefer-primordials + if (#traceId in currentSpan) { + this.#traceId = currentSpan.#traceId; + this.#parentSpanId = currentSpan.#spanId; + } else { + const context = currentSpan.spanContext(); + this.#traceId = context.traceId; + this.#parentSpanId = context.spanId; + } + } else { + if (DETERMINISTIC) { + this.#traceId = StringPrototypePadStart(String(COUNTER++), 32, "0"); + this.#spanId = StringPrototypePadStart(String(COUNTER++), 16, "0"); + } else { + const buffer = new Uint8Array(TRACE_ID_BYTES + SPAN_ID_BYTES); + op_crypto_get_random_values(buffer); + this.#traceId = TypedArrayPrototypeSubarray(buffer, 0, TRACE_ID_BYTES); + this.#spanId = TypedArrayPrototypeSubarray(buffer, TRACE_ID_BYTES); + } + } + } + + spanContext() { + if (!this.#spanContext) { + this.#spanContext = { + traceId: typeof this.#traceId === "string" + ? this.#traceId + : bytesToHex(this.#traceId), + spanId: typeof this.#spanId === "string" + ? this.#spanId + : bytesToHex(this.#spanId), + traceFlags: this.#traceFlags, + }; + } + return this.#spanContext; + } + + get parentSpanId() { + if (!this.#parentSpanIdString && this.#parentSpanId) { + if (typeof this.#parentSpanId === "string") { + this.#parentSpanIdString = this.#parentSpanId; + } else { + this.#parentSpanIdString = bytesToHex(this.#parentSpanId); + } + } + return this.#parentSpanIdString; + } + + setAttribute(name: string, value: AttributeValue) { + if (this.#recording) this.#attributes[name] = value; + return this; + } + + setAttributes(attributes: Attributes) { + if (this.#recording) ObjectAssign(this.#attributes, attributes); + return this; + } + + setStatus(status: { code: number; message?: string }) { + if (this.#recording) { + if (status.code === 0) { + this.#status = null; + } else if (status.code > 2) { + throw new Error("Invalid status code"); + } else { + this.#status = status; + } + } + return this; + } + + updateName(name: string) { + if (this.#recording) this.#name = name; + return this; + } + + addEvent(_name: never) { + // TODO(@lucacasonato): implement events + if (this.#recording) this.#droppedEventsCount += 1; + return this; + } + + addLink(_link: never) { + // TODO(@lucacasonato): implement links + if (this.#recording) this.#droppedLinksCount += 1; + return this; + } + + addLinks(links: never[]) { + // TODO(@lucacasonato): implement links + if (this.#recording) this.#droppedLinksCount += links.length; + return this; + } + + isRecording() { + return this.#recording; + } +} + +// Exporter compatible with opentelemetry js library +class SpanExporter { + export( + spans: ReadableSpan[], + resultCallback: (result: ExportResult) => void, + ) { + try { + for (let i = 0; i < spans.length; i += 1) { + const span = spans[i]; + const context = span.spanContext(); + submit( + context.spanId, + context.traceId, + context.traceFlags, + span.parentSpanId ?? null, + span, + hrToSecs(span.startTime), + hrToSecs(span.endTime), + ); + } + resultCallback({ code: 0 }); + } catch (error) { + resultCallback({ + code: 1, + error: ObjectPrototypeIsPrototypeOf(error, Error) + ? error as Error + : new Error(String(error)), + }); + } + } + + async shutdown() {} + + async forceFlush() {} +} + +const CURRENT = new AsyncVariable(); + +class Context { + #data = new SafeMap(); + + // deno-lint-ignore no-explicit-any + constructor(data?: Iterable | null | undefined) { + this.#data = data ? new SafeMap(data) : new SafeMap(); + } + + getValue(key: symbol): unknown { + return this.#data.get(key); + } + + setValue(key: symbol, value: unknown): Context { + const c = new Context(this.#data); + c.#data.set(key, value); + return c; + } + + deleteValue(key: symbol): Context { + const c = new Context(this.#data); + c.#data.delete(key); + return c; + } +} + +// TODO(lucacasonato): @opentelemetry/api defines it's own ROOT_CONTEXT +const ROOT_CONTEXT = new Context(); + +// Context manager for opentelemetry js library +class ContextManager { + active(): Context { + return CURRENT.get() ?? ROOT_CONTEXT; + } + + with ReturnType>( + context: Context, + fn: F, + thisArg?: ThisParameterType, + ...args: A + ): ReturnType { + const ctx = CURRENT.enter(context); + try { + return ReflectApply(fn, thisArg, args); + } finally { + setAsyncContext(ctx); + } + } + + // deno-lint-ignore no-explicit-any + bind any>( + context: Context, + target: T, + ): T { + return ((...args) => { + const ctx = CURRENT.enter(context); + try { + return ReflectApply(target, this, args); + } finally { + setAsyncContext(ctx); + } + }) as T; + } + + enable() { + return this; + } + + disable() { + return this; + } +} + +const otelConsoleConfig = { + ignore: 0, + capture: 1, + replace: 2, +}; + +export function bootstrap( + config: [] | [ + typeof otelConsoleConfig[keyof typeof otelConsoleConfig], + number, + ], +): void { + if (config.length === 0) return; + const { 0: consoleConfig, 1: deterministic } = config; + + TRACING_ENABLED = true; + DETERMINISTIC = deterministic === 1; + + switch (consoleConfig) { + case otelConsoleConfig.capture: + core.wrapConsole(globalThis.console, new Console(otelLog)); + break; + case otelConsoleConfig.replace: + ObjectDefineProperty( + globalThis, + "console", + core.propNonEnumerable(new Console(otelLog)), + ); + break; + default: + break; + } +} + +export const telemetry = { + SpanExporter, + ContextManager, +}; diff --git a/ext/telemetry/util.ts b/ext/telemetry/util.ts new file mode 100644 index 00000000000000..7e30d5d859acc9 --- /dev/null +++ b/ext/telemetry/util.ts @@ -0,0 +1,27 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +import { primordials } from "ext:core/mod.js"; +import type { Span } from "ext:deno_telemetry/telemetry.ts"; + +const { String, StringPrototypeSlice } = primordials; + +export function updateSpanFromRequest(span: Span, request: Request) { + span.updateName(request.method); + + span.setAttribute("http.request.method", request.method); + const url = new URL(request.url); + span.setAttribute("url.full", request.url); + span.setAttribute( + "url.scheme", + StringPrototypeSlice(url.protocol, 0, -1), + ); + span.setAttribute("url.path", url.pathname); + span.setAttribute("url.query", StringPrototypeSlice(url.search, 1)); +} + +export function updateSpanFromResponse(span: Span, response: Response) { + span.setAttribute( + "http.response.status_code", + String(response.status), + ); +} diff --git a/ext/tls/Cargo.toml b/ext/tls/Cargo.toml index 08a9802df53f19..2819dc43441b9d 100644 --- a/ext/tls/Cargo.toml +++ b/ext/tls/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_tls" -version = "0.155.0" +version = "0.167.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -21,5 +21,6 @@ rustls-pemfile.workspace = true rustls-tokio-stream.workspace = true rustls-webpki.workspace = true serde.workspace = true +thiserror.workspace = true tokio.workspace = true webpki-roots.workspace = true diff --git a/ext/tls/lib.rs b/ext/tls/lib.rs index c4d548ccf21dd8..883d2995e4b913 100644 --- a/ext/tls/lib.rs +++ b/ext/tls/lib.rs @@ -9,17 +9,12 @@ pub use rustls_tokio_stream::*; pub use webpki; pub use webpki_roots; -use deno_core::anyhow::anyhow; -use deno_core::error::custom_error; -use deno_core::error::AnyError; - use rustls::client::danger::HandshakeSignatureValid; use rustls::client::danger::ServerCertVerified; use rustls::client::danger::ServerCertVerifier; use rustls::client::WebPkiServerVerifier; use rustls::ClientConfig; use rustls::DigitallySignedStruct; -use rustls::Error; use rustls::RootCertStore; use rustls_pemfile::certs; use rustls_pemfile::ec_private_keys; @@ -35,12 +30,30 @@ use std::sync::Arc; mod tls_key; pub use tls_key::*; +#[derive(Debug, thiserror::Error)] +pub enum TlsError { + #[error(transparent)] + Rustls(#[from] rustls::Error), + #[error("Unable to add pem file to certificate store: {0}")] + UnableAddPemFileToCert(std::io::Error), + #[error("Unable to decode certificate")] + CertInvalid, + #[error("No certificates found in certificate data")] + CertsNotFound, + #[error("No keys found in key data")] + KeysNotFound, + #[error("Unable to decode key")] + KeyDecode, +} + /// Lazily resolves the root cert store. /// /// This was done because the root cert store is not needed in all cases /// and takes a bit of time to initialize. pub trait RootCertStoreProvider: Send + Sync { - fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError>; + fn get_or_try_init( + &self, + ) -> Result<&RootCertStore, deno_core::error::AnyError>; } // This extension has no runtime apis, it only exports some shared native functions. @@ -77,7 +90,7 @@ impl ServerCertVerifier for NoCertificateVerification { server_name: &rustls::pki_types::ServerName<'_>, ocsp_response: &[u8], now: rustls::pki_types::UnixTime, - ) -> Result { + ) -> Result { if self.ic_allowlist.is_empty() { return Ok(ServerCertVerified::assertion()); } @@ -89,7 +102,9 @@ impl ServerCertVerifier for NoCertificateVerification { _ => { // NOTE(bartlomieju): `ServerName` is a non-exhaustive enum // so we have this catch all errors here. - return Err(Error::General("Unknown `ServerName` variant".to_string())); + return Err(rustls::Error::General( + "Unknown `ServerName` variant".to_string(), + )); } }; if self.ic_allowlist.contains(&dns_name_or_ip_address) { @@ -110,7 +125,7 @@ impl ServerCertVerifier for NoCertificateVerification { message: &[u8], cert: &rustls::pki_types::CertificateDer, dss: &DigitallySignedStruct, - ) -> Result { + ) -> Result { if self.ic_allowlist.is_empty() { return Ok(HandshakeSignatureValid::assertion()); } @@ -126,7 +141,7 @@ impl ServerCertVerifier for NoCertificateVerification { message: &[u8], cert: &rustls::pki_types::CertificateDer, dss: &DigitallySignedStruct, - ) -> Result { + ) -> Result { if self.ic_allowlist.is_empty() { return Ok(HandshakeSignatureValid::assertion()); } @@ -178,7 +193,7 @@ pub fn create_client_config( unsafely_ignore_certificate_errors: Option>, maybe_cert_chain_and_key: TlsKeys, socket_use: SocketUse, -) -> Result { +) -> Result { if let Some(ic_allowlist) = unsafely_ignore_certificate_errors { let client_config = ClientConfig::builder() .dangerous() @@ -214,10 +229,7 @@ pub fn create_client_config( root_cert_store.add(cert)?; } Err(e) => { - return Err(anyhow!( - "Unable to add pem file to certificate store: {}", - e - )); + return Err(TlsError::UnableAddPemFileToCert(e)); } } } @@ -255,74 +267,61 @@ fn add_alpn(client: &mut ClientConfig, socket_use: SocketUse) { pub fn load_certs( reader: &mut dyn BufRead, -) -> Result>, AnyError> { +) -> Result>, TlsError> { let certs: Result, _> = certs(reader).collect(); - let certs = certs - .map_err(|_| custom_error("InvalidData", "Unable to decode certificate"))?; + let certs = certs.map_err(|_| TlsError::CertInvalid)?; if certs.is_empty() { - return Err(cert_not_found_err()); + return Err(TlsError::CertsNotFound); } Ok(certs) } -fn key_decode_err() -> AnyError { - custom_error("InvalidData", "Unable to decode key") -} - -fn key_not_found_err() -> AnyError { - custom_error("InvalidData", "No keys found in key data") -} - -fn cert_not_found_err() -> AnyError { - custom_error("InvalidData", "No certificates found in certificate data") -} - /// Starts with -----BEGIN RSA PRIVATE KEY----- fn load_rsa_keys( mut bytes: &[u8], -) -> Result>, AnyError> { +) -> Result>, TlsError> { let keys: Result, _> = rsa_private_keys(&mut bytes).collect(); - let keys = keys.map_err(|_| key_decode_err())?; + let keys = keys.map_err(|_| TlsError::KeyDecode)?; Ok(keys.into_iter().map(PrivateKeyDer::Pkcs1).collect()) } /// Starts with -----BEGIN EC PRIVATE KEY----- fn load_ec_keys( mut bytes: &[u8], -) -> Result>, AnyError> { +) -> Result>, TlsError> { let keys: Result, std::io::Error> = ec_private_keys(&mut bytes).collect(); - let keys2 = keys.map_err(|_| key_decode_err())?; + let keys2 = keys.map_err(|_| TlsError::KeyDecode)?; Ok(keys2.into_iter().map(PrivateKeyDer::Sec1).collect()) } /// Starts with -----BEGIN PRIVATE KEY----- fn load_pkcs8_keys( mut bytes: &[u8], -) -> Result>, AnyError> { +) -> Result>, TlsError> { let keys: Result, std::io::Error> = pkcs8_private_keys(&mut bytes).collect(); - let keys2 = keys.map_err(|_| key_decode_err())?; + let keys2 = keys.map_err(|_| TlsError::KeyDecode)?; Ok(keys2.into_iter().map(PrivateKeyDer::Pkcs8).collect()) } fn filter_invalid_encoding_err( - to_be_filtered: Result, -) -> Result { + to_be_filtered: Result, +) -> Result { match to_be_filtered { - Err(Error::InvalidCertificate(rustls::CertificateError::BadEncoding)) => { - Ok(HandshakeSignatureValid::assertion()) - } + Err(rustls::Error::InvalidCertificate( + rustls::CertificateError::BadEncoding, + )) => Ok(HandshakeSignatureValid::assertion()), res => res, } } pub fn load_private_keys( bytes: &[u8], -) -> Result>, AnyError> { +) -> Result>, TlsError> { let mut keys = load_rsa_keys(bytes)?; if keys.is_empty() { @@ -334,7 +333,7 @@ pub fn load_private_keys( } if keys.is_empty() { - return Err(key_not_found_err()); + return Err(TlsError::KeysNotFound); } Ok(keys) diff --git a/ext/tls/tls_key.rs b/ext/tls/tls_key.rs index 66fac86f87f9c1..b7baa604b9add6 100644 --- a/ext/tls/tls_key.rs +++ b/ext/tls/tls_key.rs @@ -11,8 +11,6 @@ //! key lookup can handle closing one end of the pair, in which case they will just //! attempt to clean up the associated resources. -use deno_core::anyhow::anyhow; -use deno_core::error::AnyError; use deno_core::futures::future::poll_fn; use deno_core::futures::future::Either; use deno_core::futures::FutureExt; @@ -33,7 +31,19 @@ use tokio::sync::oneshot; use webpki::types::CertificateDer; use webpki::types::PrivateKeyDer; -type ErrorType = Rc; +#[derive(Debug, thiserror::Error)] +pub enum TlsKeyError { + #[error(transparent)] + Rustls(#[from] rustls::Error), + #[error("Failed: {0}")] + Failed(ErrorType), + #[error(transparent)] + JoinError(#[from] tokio::task::JoinError), + #[error(transparent)] + RecvError(#[from] tokio::sync::broadcast::error::RecvError), +} + +type ErrorType = Arc>; /// A TLS certificate/private key pair. /// see https://docs.rs/rustls-pki-types/latest/rustls_pki_types/#cloning-private-keys @@ -114,7 +124,7 @@ impl TlsKeyResolver { &self, sni: String, alpn: Vec>, - ) -> Result, AnyError> { + ) -> Result, TlsKeyError> { let key = self.resolve(sni).await?; let mut tls_config = ServerConfig::builder() @@ -183,7 +193,7 @@ impl TlsKeyResolver { pub fn resolve( &self, sni: String, - ) -> impl Future> { + ) -> impl Future> { let mut cache = self.inner.cache.borrow_mut(); let mut recv = match cache.get(&sni) { None => { @@ -194,7 +204,7 @@ impl TlsKeyResolver { } Some(TlsKeyState::Resolving(recv)) => recv.resubscribe(), Some(TlsKeyState::Resolved(res)) => { - return Either::Left(ready(res.clone().map_err(|_| anyhow!("Failed")))); + return Either::Left(ready(res.clone().map_err(TlsKeyError::Failed))); } }; drop(cache); @@ -212,7 +222,7 @@ impl TlsKeyResolver { // Someone beat us to it } } - res.map_err(|_| anyhow!("Failed")) + res.map_err(TlsKeyError::Failed) }); Either::Right(async move { handle.await? }) } @@ -247,13 +257,13 @@ impl TlsKeyLookup { } /// Resolve a previously polled item. - pub fn resolve(&self, sni: String, res: Result) { + pub fn resolve(&self, sni: String, res: Result) { _ = self .pending .borrow_mut() .remove(&sni) .unwrap() - .send(res.map_err(Rc::new)); + .send(res.map_err(|e| Arc::new(e.into_boxed_str()))); } } diff --git a/ext/url/00_url.js b/ext/url/00_url.js index 577caba902f18f..ec875da768d6f9 100644 --- a/ext/url/00_url.js +++ b/ext/url/00_url.js @@ -139,7 +139,7 @@ class URLSearchParams { throw new TypeError( `${prefix}: Item ${ i + 0 - } in the parameter list does have length 2 exactly.`, + } in the parameter list does have length 2 exactly`, ); } return [pair[0], pair[1]]; diff --git a/ext/url/01_urlpattern.js b/ext/url/01_urlpattern.js index 58a6d6bce08725..6e27563089c56e 100644 --- a/ext/url/01_urlpattern.js +++ b/ext/url/01_urlpattern.js @@ -31,6 +31,7 @@ import * as webidl from "ext:deno_webidl/00_webidl.js"; import { createFilteredInspectProxy } from "ext:deno_console/01_console.js"; const _components = Symbol("components"); +const urlPatternSettings = { groupStringFallback: false }; /** * @typedef Components @@ -349,7 +350,11 @@ class URLPattern { const groups = res.groups; for (let i = 0; i < groupList.length; ++i) { // TODO(lucacasonato): this is vulnerable to override mistake - groups[groupList[i]] = match[i + 1] ?? ""; // TODO(@crowlKats): remove fallback for 2.0 + if (urlPatternSettings.groupStringFallback) { + groups[groupList[i]] = match[i + 1] ?? ""; + } else { + groups[groupList[i]] = match[i + 1]; + } } break; } @@ -422,4 +427,4 @@ webidl.converters.URLPatternOptions = webidl }, ]); -export { URLPattern }; +export { URLPattern, urlPatternSettings }; diff --git a/ext/url/Cargo.toml b/ext/url/Cargo.toml index 7f86e14456d8db..42cec915b56546 100644 --- a/ext/url/Cargo.toml +++ b/ext/url/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_url" -version = "0.168.0" +version = "0.180.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -15,6 +15,7 @@ path = "lib.rs" [dependencies] deno_core.workspace = true +thiserror.workspace = true urlpattern = "0.3.0" [dev-dependencies] diff --git a/ext/url/lib.rs b/ext/url/lib.rs index 6869d656b5d816..f8946532ae613e 100644 --- a/ext/url/lib.rs +++ b/ext/url/lib.rs @@ -15,6 +15,8 @@ use std::path::PathBuf; use crate::urlpattern::op_urlpattern_parse; use crate::urlpattern::op_urlpattern_process_match_input; +pub use urlpattern::UrlPatternError; + deno_core::extension!( deno_url, deps = [deno_webidl], diff --git a/ext/url/urlpattern.rs b/ext/url/urlpattern.rs index b6d9a13828c8d3..7d4e8ee71b06de 100644 --- a/ext/url/urlpattern.rs +++ b/ext/url/urlpattern.rs @@ -1,7 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use urlpattern::quirks; @@ -9,21 +7,23 @@ use urlpattern::quirks::MatchInput; use urlpattern::quirks::StringOrInit; use urlpattern::quirks::UrlPattern; +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +pub struct UrlPatternError(urlpattern::Error); + #[op2] #[serde] pub fn op_urlpattern_parse( #[serde] input: StringOrInit, #[string] base_url: Option, #[serde] options: urlpattern::UrlPatternOptions, -) -> Result { - let init = urlpattern::quirks::process_construct_pattern_input( - input, - base_url.as_deref(), - ) - .map_err(|e| type_error(e.to_string()))?; +) -> Result { + let init = + quirks::process_construct_pattern_input(input, base_url.as_deref()) + .map_err(UrlPatternError)?; - let pattern = urlpattern::quirks::parse_pattern(init, options) - .map_err(|e| type_error(e.to_string()))?; + let pattern = + quirks::parse_pattern(init, options).map_err(UrlPatternError)?; Ok(pattern) } @@ -33,14 +33,14 @@ pub fn op_urlpattern_parse( pub fn op_urlpattern_process_match_input( #[serde] input: StringOrInit, #[string] base_url: Option, -) -> Result, AnyError> { - let res = urlpattern::quirks::process_match_input(input, base_url.as_deref()) - .map_err(|e| type_error(e.to_string()))?; +) -> Result, UrlPatternError> { + let res = quirks::process_match_input(input, base_url.as_deref()) + .map_err(UrlPatternError)?; let (input, inputs) = match res { Some((input, inputs)) => (input, inputs), None => return Ok(None), }; - Ok(urlpattern::quirks::parse_match_input(input).map(|input| (input, inputs))) + Ok(quirks::parse_match_input(input).map(|input| (input, inputs))) } diff --git a/ext/web/02_timers.js b/ext/web/02_timers.js index 89acaca42b4485..6058febd597537 100644 --- a/ext/web/02_timers.js +++ b/ext/web/02_timers.js @@ -1,12 +1,9 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. import { core, primordials } from "ext:core/mod.js"; -import { op_defer, op_now } from "ext:core/ops"; +import { op_defer } from "ext:core/ops"; const { - Uint8Array, - Uint32Array, PromisePrototypeThen, - TypedArrayPrototypeGetBuffer, TypeError, indirectEval, ReflectApply, @@ -18,13 +15,6 @@ const { import * as webidl from "ext:deno_webidl/00_webidl.js"; -const hrU8 = new Uint8Array(8); -const hr = new Uint32Array(TypedArrayPrototypeGetBuffer(hrU8)); -function opNow() { - op_now(hrU8); - return (hr[0] * 1000 + hr[1] / 1e6); -} - // --------------------------------------------------------------------------- function checkThis(thisArg) { @@ -151,7 +141,6 @@ export { clearInterval, clearTimeout, defer, - opNow, refTimer, setImmediate, setInterval, diff --git a/ext/web/06_streams.js b/ext/web/06_streams.js index 688e87a0e51b49..e673ee2bb4cc16 100644 --- a/ext/web/06_streams.js +++ b/ext/web/06_streams.js @@ -70,7 +70,6 @@ const { String, Symbol, SymbolAsyncIterator, - SymbolIterator, SymbolFor, TypeError, TypedArrayPrototypeGetBuffer, @@ -2927,7 +2926,7 @@ function readableStreamPipeTo( } /** - * @param {ReadableStreamGenericReader | ReadableStreamBYOBReader} reader + * @param {ReadableStreamGenericReader | ReadableStreamBYOBReader} reader * @param {any} reason * @returns {Promise} */ @@ -2960,7 +2959,7 @@ function readableStreamReaderGenericInitialize(reader, stream) { /** * @template R - * @param {ReadableStreamGenericReader | ReadableStreamBYOBReader} reader + * @param {ReadableStreamGenericReader | ReadableStreamBYOBReader} reader */ function readableStreamReaderGenericRelease(reader) { const stream = reader[_stream]; @@ -5088,34 +5087,6 @@ function initializeCountSizeFunction(globalObject) { WeakMapPrototypeSet(countSizeFunctionWeakMap, globalObject, size); } -// Ref: https://tc39.es/ecma262/#sec-getiterator -function getAsyncOrSyncIterator(obj) { - let iterator; - if (obj[SymbolAsyncIterator] != null) { - iterator = obj[SymbolAsyncIterator](); - if (!isObject(iterator)) { - throw new TypeError( - "[Symbol.asyncIterator] returned a non-object value", - ); - } - } else if (obj[SymbolIterator] != null) { - iterator = obj[SymbolIterator](); - if (!isObject(iterator)) { - throw new TypeError("[Symbol.iterator] returned a non-object value"); - } - } else { - throw new TypeError("No iterator found"); - } - if (typeof iterator.next !== "function") { - throw new TypeError("iterator.next is not a function"); - } - return iterator; -} - -function isObject(x) { - return (typeof x === "object" && x != null) || typeof x === "function"; -} - const _resourceBacking = Symbol("[[resourceBacking]]"); // This distinction exists to prevent unrefable streams being used in // regular fast streams that are unaware of refability @@ -5201,21 +5172,22 @@ class ReadableStream { } static from(asyncIterable) { + const prefix = "Failed to execute 'ReadableStream.from'"; webidl.requiredArguments( arguments.length, 1, - "Failed to execute 'ReadableStream.from'", + prefix, ); - asyncIterable = webidl.converters.any(asyncIterable); - - const iterator = getAsyncOrSyncIterator(asyncIterable); + asyncIterable = webidl.converters["async iterable"]( + asyncIterable, + prefix, + "Argument 1", + ); + const iter = asyncIterable.open(); const stream = createReadableStream(noop, async () => { // deno-lint-ignore prefer-primordials - const res = await iterator.next(); - if (!isObject(res)) { - throw new TypeError("iterator.next value is not an object"); - } + const res = await iter.next(); if (res.done) { readableStreamDefaultControllerClose(stream[_controller]); } else { @@ -5225,17 +5197,8 @@ class ReadableStream { ); } }, async (reason) => { - if (iterator.return == null) { - return undefined; - } else { - // deno-lint-ignore prefer-primordials - const res = await iterator.return(reason); - if (!isObject(res)) { - throw new TypeError("iterator.return value is not an object"); - } else { - return undefined; - } - } + // deno-lint-ignore prefer-primordials + await iter.return(reason); }, 0); return stream; } @@ -6896,6 +6859,10 @@ webidl.converters.StreamPipeOptions = webidl { key: "signal", converter: webidl.converters.AbortSignal }, ]); +webidl.converters["async iterable"] = webidl.createAsyncIterableConverter( + webidl.converters.any, +); + internals.resourceForReadableStream = resourceForReadableStream; export { diff --git a/ext/web/06_streams_types.d.ts b/ext/web/06_streams_types.d.ts index e04f568d26644a..fe05ee6e65b167 100644 --- a/ext/web/06_streams_types.d.ts +++ b/ext/web/06_streams_types.d.ts @@ -60,8 +60,8 @@ interface VoidFunction { (): void; } -interface ReadableStreamGenericReader { - readonly closed: Promise; +interface ReadableStreamGenericReader { + readonly closed: Promise; // deno-lint-ignore no-explicit-any cancel(reason?: any): Promise; } diff --git a/ext/web/13_message_port.js b/ext/web/13_message_port.js index 04697d6aa83614..cf72c43e6ff291 100644 --- a/ext/web/13_message_port.js +++ b/ext/web/13_message_port.js @@ -22,6 +22,7 @@ const { Symbol, SymbolFor, SymbolIterator, + PromiseResolve, SafeArrayIterator, TypeError, } = primordials; @@ -41,7 +42,10 @@ import { import { isDetachedBuffer } from "./06_streams.js"; import { DOMException } from "./01_dom_exception.js"; -let messageEventListenerCount = 0; +// counter of how many message ports are actively refed +// either due to the existence of "message" event listeners or +// explicit calls to ref/unref (in the case of node message ports) +let refedMessagePortsCount = 0; class MessageChannel { /** @type {MessagePort} */ @@ -93,6 +97,7 @@ const MessagePortReceiveMessageOnPortSymbol = Symbol( ); const _enabled = Symbol("enabled"); const _refed = Symbol("refed"); +const _messageEventListenerCount = Symbol("messageEventListenerCount"); const nodeWorkerThreadCloseCb = Symbol("nodeWorkerThreadCloseCb"); const nodeWorkerThreadCloseCbInvoked = Symbol("nodeWorkerThreadCloseCbInvoked"); export const refMessagePort = Symbol("refMessagePort"); @@ -109,6 +114,9 @@ function createMessagePort(id) { port[core.hostObjectBrand] = core.hostObjectBrand; setEventTargetData(port); port[_id] = id; + port[_enabled] = false; + port[_messageEventListenerCount] = 0; + port[_refed] = false; return port; } @@ -122,12 +130,18 @@ function nodeWorkerThreadMaybeInvokeCloseCb(port) { } } +const _isRefed = Symbol("isRefed"); +const _dataPromise = Symbol("dataPromise"); + class MessagePort extends EventTarget { /** @type {number | null} */ [_id] = null; /** @type {boolean} */ [_enabled] = false; [_refed] = false; + /** @type {Promise | undefined} */ + [_dataPromise] = undefined; + [_messageEventListenerCount] = 0; constructor() { super(); @@ -193,24 +207,21 @@ class MessagePort extends EventTarget { this[_enabled] = true; while (true) { if (this[_id] === null) break; - // Exit if no message event listeners are present in Node compat mode. - if ( - typeof this[nodeWorkerThreadCloseCb] == "function" && - messageEventListenerCount === 0 - ) break; let data; try { - data = await op_message_port_recv_message( + this[_dataPromise] = op_message_port_recv_message( this[_id], ); + if ( + typeof this[nodeWorkerThreadCloseCb] === "function" && + !this[_refed] + ) { + core.unrefOpPromise(this[_dataPromise]); + } + data = await this[_dataPromise]; + this[_dataPromise] = undefined; } catch (err) { if (ObjectPrototypeIsPrototypeOf(InterruptedPrototype, err)) { - // If we were interrupted, check if the interruption is coming - // from `receiveMessageOnPort` API from Node compat, if so, continue. - if (this[MessagePortReceiveMessageOnPortSymbol]) { - this[MessagePortReceiveMessageOnPortSymbol] = false; - continue; - } break; } nodeWorkerThreadMaybeInvokeCloseCb(this); @@ -246,12 +257,26 @@ class MessagePort extends EventTarget { } [refMessagePort](ref) { - if (ref && !this[_refed]) { - this[_refed] = true; - messageEventListenerCount++; - } else if (!ref && this[_refed]) { - this[_refed] = false; - messageEventListenerCount = 0; + if (ref) { + if (!this[_refed]) { + refedMessagePortsCount++; + if ( + this[_dataPromise] + ) { + core.refOpPromise(this[_dataPromise]); + } + this[_refed] = true; + } + } else if (!ref) { + if (this[_refed]) { + refedMessagePortsCount--; + if ( + this[_dataPromise] + ) { + core.unrefOpPromise(this[_dataPromise]); + } + this[_refed] = false; + } } } @@ -266,15 +291,20 @@ class MessagePort extends EventTarget { removeEventListener(...args) { if (args[0] == "message") { - messageEventListenerCount--; + if (--this[_messageEventListenerCount] === 0 && this[_refed]) { + refedMessagePortsCount--; + this[_refed] = false; + } } super.removeEventListener(...new SafeArrayIterator(args)); } addEventListener(...args) { if (args[0] == "message") { - messageEventListenerCount++; - if (!this[_refed]) this[_refed] = true; + if (++this[_messageEventListenerCount] === 1 && !this[_refed]) { + refedMessagePortsCount++; + this[_refed] = true; + } } super.addEventListener(...new SafeArrayIterator(args)); } @@ -295,7 +325,17 @@ class MessagePort extends EventTarget { } defineEventHandler(MessagePort.prototype, "message", function (self) { - self.start(); + if (self[nodeWorkerThreadCloseCb]) { + (async () => { + // delay `start()` until he end of this event loop turn, to give `receiveMessageOnPort` + // a chance to receive a message first. this is primarily to resolve an issue with + // a pattern used in `npm:piscina` that results in an indefinite hang + await PromiseResolve(); + self.start(); + })(); + } else { + self.start(); + } }); defineEventHandler(MessagePort.prototype, "messageerror"); @@ -463,12 +503,12 @@ function structuredClone(value, options) { export { deserializeJsMessageData, MessageChannel, - messageEventListenerCount, MessagePort, MessagePortIdSymbol, MessagePortPrototype, MessagePortReceiveMessageOnPortSymbol, nodeWorkerThreadCloseCb, + refedMessagePortsCount, serializeJsMessageData, structuredClone, }; diff --git a/ext/web/15_performance.js b/ext/web/15_performance.js index 2ac92d16cbaa09..f23e8512466348 100644 --- a/ext/web/15_performance.js +++ b/ext/web/15_performance.js @@ -1,6 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. import { primordials } from "ext:core/mod.js"; +import { op_now, op_time_origin } from "ext:core/ops"; const { ArrayPrototypeFilter, ArrayPrototypePush, @@ -10,19 +11,34 @@ const { Symbol, SymbolFor, TypeError, + TypedArrayPrototypeGetBuffer, + Uint8Array, + Uint32Array, } = primordials; import * as webidl from "ext:deno_webidl/00_webidl.js"; import { structuredClone } from "./02_structured_clone.js"; import { createFilteredInspectProxy } from "ext:deno_console/01_console.js"; import { EventTarget } from "./02_event.js"; -import { opNow } from "./02_timers.js"; import { DOMException } from "./01_dom_exception.js"; const illegalConstructorKey = Symbol("illegalConstructorKey"); let performanceEntries = []; let timeOrigin; +const hrU8 = new Uint8Array(8); +const hr = new Uint32Array(TypedArrayPrototypeGetBuffer(hrU8)); + +function setTimeOrigin() { + op_time_origin(hrU8); + timeOrigin = hr[0] * 1000 + hr[1] / 1e6; +} + +function now() { + op_now(hrU8); + return hr[0] * 1000 + hr[1] / 1e6; +} + webidl.converters["PerformanceMarkOptions"] = webidl .createDictionaryConverter( "PerformanceMarkOptions", @@ -90,10 +106,6 @@ webidl.converters["DOMString or PerformanceMeasureOptions"] = ( return webidl.converters.DOMString(V, prefix, context, opts); }; -function setTimeOrigin(origin) { - timeOrigin = origin; -} - function findMostRecent( name, type, @@ -135,8 +147,6 @@ function filterByNameType( ); } -const now = opNow; - const _name = Symbol("[[name]]"); const _entryType = Symbol("[[entryType]]"); const _startTime = Symbol("[[startTime]]"); diff --git a/ext/web/Cargo.toml b/ext/web/Cargo.toml index 7fd12632d7668b..43c6d610538509 100644 --- a/ext/web/Cargo.toml +++ b/ext/web/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_web" -version = "0.199.0" +version = "0.211.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -23,6 +23,7 @@ encoding_rs.workspace = true flate2 = { workspace = true, features = ["default"] } futures.workspace = true serde = "1.0.149" +thiserror.workspace = true tokio.workspace = true uuid = { workspace = true, features = ["serde"] } diff --git a/ext/web/blob.rs b/ext/web/blob.rs index 392f36acb8f9b5..bc64a0f27eba17 100644 --- a/ext/web/blob.rs +++ b/ext/web/blob.rs @@ -7,8 +7,6 @@ use std::rc::Rc; use std::sync::Arc; use async_trait::async_trait; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::parking_lot::Mutex; use deno_core::url::Url; @@ -19,6 +17,18 @@ use serde::Deserialize; use serde::Serialize; use uuid::Uuid; +#[derive(Debug, thiserror::Error)] +pub enum BlobError { + #[error("Blob part not found")] + BlobPartNotFound, + #[error("start + len can not be larger than blob part size")] + SizeLargerThanBlobPart, + #[error("Blob URLs are not supported in this context")] + BlobURLsNotSupported, + #[error(transparent)] + Url(#[from] deno_core::url::ParseError), +} + use crate::Location; pub type PartMap = HashMap>; @@ -96,18 +106,18 @@ pub struct Blob { impl Blob { // TODO(lucacsonato): this should be a stream! - pub async fn read_all(&self) -> Result, AnyError> { + pub async fn read_all(&self) -> Vec { let size = self.size(); let mut bytes = Vec::with_capacity(size); for part in &self.parts { - let chunk = part.read().await?; + let chunk = part.read().await; bytes.extend_from_slice(chunk); } assert_eq!(bytes.len(), size); - Ok(bytes) + bytes } fn size(&self) -> usize { @@ -122,7 +132,7 @@ impl Blob { #[async_trait] pub trait BlobPart: Debug { // TODO(lucacsonato): this should be a stream! - async fn read(&self) -> Result<&[u8], AnyError>; + async fn read(&self) -> &[u8]; fn size(&self) -> usize; } @@ -137,8 +147,8 @@ impl From> for InMemoryBlobPart { #[async_trait] impl BlobPart for InMemoryBlobPart { - async fn read(&self) -> Result<&[u8], AnyError> { - Ok(&self.0) + async fn read(&self) -> &[u8] { + &self.0 } fn size(&self) -> usize { @@ -155,9 +165,9 @@ pub struct SlicedBlobPart { #[async_trait] impl BlobPart for SlicedBlobPart { - async fn read(&self) -> Result<&[u8], AnyError> { - let original = self.part.read().await?; - Ok(&original[self.start..self.start + self.len]) + async fn read(&self) -> &[u8] { + let original = self.part.read().await; + &original[self.start..self.start + self.len] } fn size(&self) -> usize { @@ -189,19 +199,17 @@ pub fn op_blob_slice_part( state: &mut OpState, #[serde] id: Uuid, #[serde] options: SliceOptions, -) -> Result { +) -> Result { let blob_store = state.borrow::>(); let part = blob_store .get_part(&id) - .ok_or_else(|| type_error("Blob part not found"))?; + .ok_or(BlobError::BlobPartNotFound)?; let SliceOptions { start, len } = options; let size = part.size(); if start + len > size { - return Err(type_error( - "start + len can not be larger than blob part size", - )); + return Err(BlobError::SizeLargerThanBlobPart); } let sliced_part = SlicedBlobPart { part, start, len }; @@ -215,14 +223,14 @@ pub fn op_blob_slice_part( pub async fn op_blob_read_part( state: Rc>, #[serde] id: Uuid, -) -> Result { +) -> Result { let part = { let state = state.borrow(); let blob_store = state.borrow::>(); blob_store.get_part(&id) } - .ok_or_else(|| type_error("Blob part not found"))?; - let buf = part.read().await?; + .ok_or(BlobError::BlobPartNotFound)?; + let buf = part.read().await; Ok(ToJsBuffer::from(buf.to_vec())) } @@ -238,13 +246,13 @@ pub fn op_blob_create_object_url( state: &mut OpState, #[string] media_type: String, #[serde] part_ids: Vec, -) -> Result { +) -> Result { let mut parts = Vec::with_capacity(part_ids.len()); let blob_store = state.borrow::>(); for part_id in part_ids { let part = blob_store .get_part(&part_id) - .ok_or_else(|| type_error("Blob part not found"))?; + .ok_or(BlobError::BlobPartNotFound)?; parts.push(part); } @@ -263,7 +271,7 @@ pub fn op_blob_create_object_url( pub fn op_blob_revoke_object_url( state: &mut OpState, #[string] url: &str, -) -> Result<(), AnyError> { +) -> Result<(), BlobError> { let url = Url::parse(url)?; let blob_store = state.borrow::>(); blob_store.remove_object_url(&url); @@ -287,15 +295,15 @@ pub struct ReturnBlobPart { pub fn op_blob_from_object_url( state: &mut OpState, #[string] url: String, -) -> Result, AnyError> { +) -> Result, BlobError> { let url = Url::parse(&url)?; if url.scheme() != "blob" { return Ok(None); } - let blob_store = state.try_borrow::>().ok_or_else(|| { - type_error("Blob URLs are not supported in this context.") - })?; + let blob_store = state + .try_borrow::>() + .ok_or(BlobError::BlobURLsNotSupported)?; if let Some(blob) = blob_store.get_object_url(url) { let parts = blob .parts diff --git a/ext/web/compression.rs b/ext/web/compression.rs index b9ae12ef17325c..6967009915d419 100644 --- a/ext/web/compression.rs +++ b/ext/web/compression.rs @@ -1,7 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use flate2::write::DeflateDecoder; use flate2::write::DeflateEncoder; @@ -13,6 +11,18 @@ use flate2::Compression; use std::cell::RefCell; use std::io::Write; +#[derive(Debug, thiserror::Error)] +pub enum CompressionError { + #[error("Unsupported format")] + UnsupportedFormat, + #[error("resource is closed")] + ResourceClosed, + #[error(transparent)] + IoTypeError(std::io::Error), + #[error(transparent)] + Io(std::io::Error), +} + #[derive(Debug)] struct CompressionResource(RefCell>); @@ -34,7 +44,7 @@ enum Inner { pub fn op_compression_new( #[string] format: &str, is_decoder: bool, -) -> Result { +) -> Result { let w = Vec::new(); let inner = match (format, is_decoder) { ("deflate", true) => Inner::DeflateDecoder(ZlibDecoder::new(w)), @@ -49,7 +59,7 @@ pub fn op_compression_new( ("gzip", false) => { Inner::GzEncoder(GzEncoder::new(w, Compression::default())) } - _ => return Err(type_error("Unsupported format")), + _ => return Err(CompressionError::UnsupportedFormat), }; Ok(CompressionResource(RefCell::new(Some(inner)))) } @@ -59,40 +69,38 @@ pub fn op_compression_new( pub fn op_compression_write( #[cppgc] resource: &CompressionResource, #[anybuffer] input: &[u8], -) -> Result, AnyError> { +) -> Result, CompressionError> { let mut inner = resource.0.borrow_mut(); - let inner = inner - .as_mut() - .ok_or_else(|| type_error("resource is closed"))?; + let inner = inner.as_mut().ok_or(CompressionError::ResourceClosed)?; let out: Vec = match &mut *inner { Inner::DeflateDecoder(d) => { - d.write_all(input).map_err(|e| type_error(e.to_string()))?; - d.flush()?; + d.write_all(input).map_err(CompressionError::IoTypeError)?; + d.flush().map_err(CompressionError::Io)?; d.get_mut().drain(..) } Inner::DeflateEncoder(d) => { - d.write_all(input).map_err(|e| type_error(e.to_string()))?; - d.flush()?; + d.write_all(input).map_err(CompressionError::IoTypeError)?; + d.flush().map_err(CompressionError::Io)?; d.get_mut().drain(..) } Inner::DeflateRawDecoder(d) => { - d.write_all(input).map_err(|e| type_error(e.to_string()))?; - d.flush()?; + d.write_all(input).map_err(CompressionError::IoTypeError)?; + d.flush().map_err(CompressionError::Io)?; d.get_mut().drain(..) } Inner::DeflateRawEncoder(d) => { - d.write_all(input).map_err(|e| type_error(e.to_string()))?; - d.flush()?; + d.write_all(input).map_err(CompressionError::IoTypeError)?; + d.flush().map_err(CompressionError::Io)?; d.get_mut().drain(..) } Inner::GzDecoder(d) => { - d.write_all(input).map_err(|e| type_error(e.to_string()))?; - d.flush()?; + d.write_all(input).map_err(CompressionError::IoTypeError)?; + d.flush().map_err(CompressionError::Io)?; d.get_mut().drain(..) } Inner::GzEncoder(d) => { - d.write_all(input).map_err(|e| type_error(e.to_string()))?; - d.flush()?; + d.write_all(input).map_err(CompressionError::IoTypeError)?; + d.flush().map_err(CompressionError::Io)?; d.get_mut().drain(..) } } @@ -105,27 +113,27 @@ pub fn op_compression_write( pub fn op_compression_finish( #[cppgc] resource: &CompressionResource, report_errors: bool, -) -> Result, AnyError> { +) -> Result, CompressionError> { let inner = resource .0 .borrow_mut() .take() - .ok_or_else(|| type_error("resource is closed"))?; + .ok_or(CompressionError::ResourceClosed)?; let out = match inner { Inner::DeflateDecoder(d) => { - d.finish().map_err(|e| type_error(e.to_string())) + d.finish().map_err(CompressionError::IoTypeError) } Inner::DeflateEncoder(d) => { - d.finish().map_err(|e| type_error(e.to_string())) + d.finish().map_err(CompressionError::IoTypeError) } Inner::DeflateRawDecoder(d) => { - d.finish().map_err(|e| type_error(e.to_string())) + d.finish().map_err(CompressionError::IoTypeError) } Inner::DeflateRawEncoder(d) => { - d.finish().map_err(|e| type_error(e.to_string())) + d.finish().map_err(CompressionError::IoTypeError) } - Inner::GzDecoder(d) => d.finish().map_err(|e| type_error(e.to_string())), - Inner::GzEncoder(d) => d.finish().map_err(|e| type_error(e.to_string())), + Inner::GzDecoder(d) => d.finish().map_err(CompressionError::IoTypeError), + Inner::GzEncoder(d) => d.finish().map_err(CompressionError::IoTypeError), }; match out { Err(err) => { diff --git a/ext/web/lib.deno_web.d.ts b/ext/web/lib.deno_web.d.ts index 2ad97ac7d1eef2..8aafbad535923a 100644 --- a/ext/web/lib.deno_web.d.ts +++ b/ext/web/lib.deno_web.d.ts @@ -882,7 +882,7 @@ declare var ReadableStream: { strategy?: QueuingStrategy, ): ReadableStream; from( - asyncIterable: AsyncIterable | Iterable>, + asyncIterable: AsyncIterable | Iterable> & object, ): ReadableStream; }; diff --git a/ext/web/lib.rs b/ext/web/lib.rs index 3977379a5d82a0..af0fc2c276563e 100644 --- a/ext/web/lib.rs +++ b/ext/web/lib.rs @@ -6,9 +6,6 @@ mod message_port; mod stream_resource; mod timers; -use deno_core::error::range_error; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::url::Url; use deno_core::v8; @@ -22,10 +19,14 @@ use encoding_rs::DecoderResult; use encoding_rs::Encoding; use std::borrow::Cow; use std::cell::RefCell; -use std::fmt; use std::path::PathBuf; use std::sync::Arc; +pub use blob::BlobError; +pub use compression::CompressionError; +pub use message_port::MessagePortError; +pub use stream_resource::StreamResourceError; + use crate::blob::op_blob_create_object_url; use crate::blob::op_blob_create_part; use crate::blob::op_blob_from_object_url; @@ -51,7 +52,8 @@ pub use crate::message_port::Transferable; use crate::timers::op_defer; use crate::timers::op_now; -use crate::timers::StartTime; +use crate::timers::op_time_origin; +pub use crate::timers::StartTime; pub use crate::timers::TimersPermission; deno_core::extension!(deno_web, @@ -83,6 +85,7 @@ deno_core::extension!(deno_web, compression::op_compression_write, compression::op_compression_finish, op_now

, + op_time_origin

, op_defer, stream_resource::op_readable_stream_resource_allocate, stream_resource::op_readable_stream_resource_allocate_sized, @@ -122,13 +125,31 @@ deno_core::extension!(deno_web, if let Some(location) = options.maybe_location { state.put(Location(location)); } - state.put(StartTime::now()); + state.put(StartTime::default()); } ); +#[derive(Debug, thiserror::Error)] +pub enum WebError { + #[error("Failed to decode base64")] + Base64Decode, + #[error("The encoding label provided ('{0}') is invalid.")] + InvalidEncodingLabel(String), + #[error("buffer exceeds maximum length")] + BufferTooLong, + #[error("Value too large to decode")] + ValueTooLarge, + #[error("Provided buffer too small")] + BufferTooSmall, + #[error("The encoded data is not valid")] + DataInvalid, + #[error(transparent)] + DataError(#[from] v8::DataError), +} + #[op2] #[serde] -fn op_base64_decode(#[string] input: String) -> Result { +fn op_base64_decode(#[string] input: String) -> Result { let mut s = input.into_bytes(); let decoded_len = forgiving_base64_decode_inplace(&mut s)?; s.truncate(decoded_len); @@ -137,7 +158,7 @@ fn op_base64_decode(#[string] input: String) -> Result { #[op2] #[serde] -fn op_base64_atob(#[serde] mut s: ByteString) -> Result { +fn op_base64_atob(#[serde] mut s: ByteString) -> Result { let decoded_len = forgiving_base64_decode_inplace(&mut s)?; s.truncate(decoded_len); Ok(s) @@ -147,11 +168,9 @@ fn op_base64_atob(#[serde] mut s: ByteString) -> Result { #[inline] fn forgiving_base64_decode_inplace( input: &mut [u8], -) -> Result { - let error = - || DomExceptionInvalidCharacterError::new("Failed to decode base64"); - let decoded = - base64_simd::forgiving_decode_inplace(input).map_err(|_| error())?; +) -> Result { + let decoded = base64_simd::forgiving_decode_inplace(input) + .map_err(|_| WebError::Base64Decode)?; Ok(decoded.len()) } @@ -177,13 +196,9 @@ fn forgiving_base64_encode(s: &[u8]) -> String { #[string] fn op_encoding_normalize_label( #[string] label: String, -) -> Result { +) -> Result { let encoding = Encoding::for_label_no_replacement(label.as_bytes()) - .ok_or_else(|| { - range_error(format!( - "The encoding label provided ('{label}') is invalid." - )) - })?; + .ok_or(WebError::InvalidEncodingLabel(label))?; Ok(encoding.name().to_lowercase()) } @@ -192,7 +207,7 @@ fn op_encoding_decode_utf8<'a>( scope: &mut v8::HandleScope<'a>, #[anybuffer] zero_copy: &[u8], ignore_bom: bool, -) -> Result, AnyError> { +) -> Result, WebError> { let buf = &zero_copy; let buf = if !ignore_bom @@ -216,7 +231,7 @@ fn op_encoding_decode_utf8<'a>( // - https://github.com/v8/v8/blob/d68fb4733e39525f9ff0a9222107c02c28096e2a/include/v8.h#L3277-L3278 match v8::String::new_from_utf8(scope, buf, v8::NewStringType::Normal) { Some(text) => Ok(text), - None => Err(type_error("buffer exceeds maximum length")), + None => Err(WebError::BufferTooLong), } } @@ -227,12 +242,9 @@ fn op_encoding_decode_single( #[string] label: String, fatal: bool, ignore_bom: bool, -) -> Result { - let encoding = Encoding::for_label(label.as_bytes()).ok_or_else(|| { - range_error(format!( - "The encoding label provided ('{label}') is invalid." - )) - })?; +) -> Result { + let encoding = Encoding::for_label(label.as_bytes()) + .ok_or(WebError::InvalidEncodingLabel(label))?; let mut decoder = if ignore_bom { encoding.new_decoder_without_bom_handling() @@ -242,7 +254,7 @@ fn op_encoding_decode_single( let max_buffer_length = decoder .max_utf16_buffer_length(data.len()) - .ok_or_else(|| range_error("Value too large to decode."))?; + .ok_or(WebError::ValueTooLarge)?; let mut output = vec![0; max_buffer_length]; @@ -254,12 +266,8 @@ fn op_encoding_decode_single( output.truncate(written); Ok(output.into()) } - DecoderResult::OutputFull => { - Err(range_error("Provided buffer too small.")) - } - DecoderResult::Malformed(_, _) => { - Err(type_error("The encoded data is not valid.")) - } + DecoderResult::OutputFull => Err(WebError::BufferTooSmall), + DecoderResult::Malformed(_, _) => Err(WebError::DataInvalid), } } else { let (result, _, written, _) = @@ -269,7 +277,7 @@ fn op_encoding_decode_single( output.truncate(written); Ok(output.into()) } - CoderResult::OutputFull => Err(range_error("Provided buffer too small.")), + CoderResult::OutputFull => Err(WebError::BufferTooSmall), } } } @@ -280,12 +288,9 @@ fn op_encoding_new_decoder( #[string] label: &str, fatal: bool, ignore_bom: bool, -) -> Result { - let encoding = Encoding::for_label(label.as_bytes()).ok_or_else(|| { - range_error(format!( - "The encoding label provided ('{label}') is invalid." - )) - })?; +) -> Result { + let encoding = Encoding::for_label(label.as_bytes()) + .ok_or_else(|| WebError::InvalidEncodingLabel(label.to_string()))?; let decoder = if ignore_bom { encoding.new_decoder_without_bom_handling() @@ -305,13 +310,13 @@ fn op_encoding_decode( #[anybuffer] data: &[u8], #[cppgc] resource: &TextDecoderResource, stream: bool, -) -> Result { +) -> Result { let mut decoder = resource.decoder.borrow_mut(); let fatal = resource.fatal; let max_buffer_length = decoder .max_utf16_buffer_length(data.len()) - .ok_or_else(|| range_error("Value too large to decode."))?; + .ok_or(WebError::ValueTooLarge)?; let mut output = vec![0; max_buffer_length]; @@ -323,12 +328,8 @@ fn op_encoding_decode( output.truncate(written); Ok(output.into()) } - DecoderResult::OutputFull => { - Err(range_error("Provided buffer too small.")) - } - DecoderResult::Malformed(_, _) => { - Err(type_error("The encoded data is not valid.")) - } + DecoderResult::OutputFull => Err(WebError::BufferTooSmall), + DecoderResult::Malformed(_, _) => Err(WebError::DataInvalid), } } else { let (result, _, written, _) = @@ -338,7 +339,7 @@ fn op_encoding_decode( output.truncate(written); Ok(output.into()) } - CoderResult::OutputFull => Err(range_error("Provided buffer too small.")), + CoderResult::OutputFull => Err(WebError::BufferTooSmall), } } } @@ -356,7 +357,7 @@ fn op_encoding_encode_into( input: v8::Local, #[buffer] buffer: &mut [u8], #[buffer] out_buf: &mut [u32], -) -> Result<(), AnyError> { +) -> Result<(), WebError> { let s = v8::Local::::try_from(input)?; let mut nchars = 0; @@ -414,53 +415,4 @@ pub fn get_declaration() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("lib.deno_web.d.ts") } -#[derive(Debug)] -pub struct DomExceptionQuotaExceededError { - pub msg: String, -} - -impl DomExceptionQuotaExceededError { - pub fn new(msg: &str) -> Self { - DomExceptionQuotaExceededError { - msg: msg.to_string(), - } - } -} - -#[derive(Debug)] -pub struct DomExceptionInvalidCharacterError { - pub msg: String, -} - -impl DomExceptionInvalidCharacterError { - pub fn new(msg: &str) -> Self { - DomExceptionInvalidCharacterError { - msg: msg.to_string(), - } - } -} - -impl fmt::Display for DomExceptionQuotaExceededError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.pad(&self.msg) - } -} -impl fmt::Display for DomExceptionInvalidCharacterError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.pad(&self.msg) - } -} - -impl std::error::Error for DomExceptionQuotaExceededError {} - -impl std::error::Error for DomExceptionInvalidCharacterError {} - -pub fn get_error_class_name(e: &AnyError) -> Option<&'static str> { - e.downcast_ref::() - .map(|_| "DOMExceptionQuotaExceededError") - .or_else(|| { - e.downcast_ref::() - .map(|_| "DOMExceptionInvalidCharacterError") - }) -} pub struct Location(pub Url); diff --git a/ext/web/message_port.rs b/ext/web/message_port.rs index c069037f81f9f9..1a4a09073d176d 100644 --- a/ext/web/message_port.rs +++ b/ext/web/message_port.rs @@ -4,8 +4,6 @@ use std::borrow::Cow; use std::cell::RefCell; use std::rc::Rc; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::CancelFuture; @@ -23,6 +21,20 @@ use tokio::sync::mpsc::unbounded_channel; use tokio::sync::mpsc::UnboundedReceiver; use tokio::sync::mpsc::UnboundedSender; +#[derive(Debug, thiserror::Error)] +pub enum MessagePortError { + #[error("Invalid message port transfer")] + InvalidTransfer, + #[error("Message port is not ready for transfer")] + NotReady, + #[error("Can not transfer self message port")] + TransferSelf, + #[error(transparent)] + Canceled(#[from] deno_core::Canceled), + #[error(transparent)] + Resource(deno_core::error::AnyError), +} + pub enum Transferable { MessagePort(MessagePort), ArrayBuffer(u32), @@ -40,7 +52,7 @@ impl MessagePort { &self, state: &mut OpState, data: JsMessageData, - ) -> Result<(), AnyError> { + ) -> Result<(), MessagePortError> { let transferables = deserialize_js_transferables(state, data.transferables)?; @@ -56,7 +68,7 @@ impl MessagePort { pub async fn recv( &self, state: Rc>, - ) -> Result, AnyError> { + ) -> Result, MessagePortError> { let rx = &self.rx; let maybe_data = poll_fn(|cx| { @@ -147,7 +159,7 @@ pub enum JsTransferable { pub fn deserialize_js_transferables( state: &mut OpState, js_transferables: Vec, -) -> Result, AnyError> { +) -> Result, MessagePortError> { let mut transferables = Vec::with_capacity(js_transferables.len()); for js_transferable in js_transferables { match js_transferable { @@ -155,10 +167,10 @@ pub fn deserialize_js_transferables( let resource = state .resource_table .take::(id) - .map_err(|_| type_error("Invalid message port transfer"))?; + .map_err(|_| MessagePortError::InvalidTransfer)?; resource.cancel.cancel(); - let resource = Rc::try_unwrap(resource) - .map_err(|_| type_error("Message port is not ready for transfer"))?; + let resource = + Rc::try_unwrap(resource).map_err(|_| MessagePortError::NotReady)?; transferables.push(Transferable::MessagePort(resource.port)); } JsTransferable::ArrayBuffer(id) => { @@ -202,16 +214,19 @@ pub fn op_message_port_post_message( state: &mut OpState, #[smi] rid: ResourceId, #[serde] data: JsMessageData, -) -> Result<(), AnyError> { +) -> Result<(), MessagePortError> { for js_transferable in &data.transferables { if let JsTransferable::MessagePort(id) = js_transferable { if *id == rid { - return Err(type_error("Can not transfer self message port")); + return Err(MessagePortError::TransferSelf); } } } - let resource = state.resource_table.get::(rid)?; + let resource = state + .resource_table + .get::(rid) + .map_err(MessagePortError::Resource)?; resource.port.send(state, data) } @@ -220,7 +235,7 @@ pub fn op_message_port_post_message( pub async fn op_message_port_recv_message( state: Rc>, #[smi] rid: ResourceId, -) -> Result, AnyError> { +) -> Result, MessagePortError> { let resource = { let state = state.borrow(); match state.resource_table.get::(rid) { @@ -237,9 +252,11 @@ pub async fn op_message_port_recv_message( pub fn op_message_port_recv_message_sync( state: &mut OpState, // Rc>, #[smi] rid: ResourceId, -) -> Result, AnyError> { - let resource = state.resource_table.get::(rid)?; - resource.cancel.cancel(); +) -> Result, MessagePortError> { + let resource = state + .resource_table + .get::(rid) + .map_err(MessagePortError::Resource)?; let mut rx = resource.port.rx.borrow_mut(); match rx.try_recv() { diff --git a/ext/web/stream_resource.rs b/ext/web/stream_resource.rs index 78487883b6fcda..c44a385ea9d59d 100644 --- a/ext/web/stream_resource.rs +++ b/ext/web/stream_resource.rs @@ -1,7 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use bytes::BytesMut; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::external; use deno_core::op2; use deno_core::serde_v8::V8Slice; @@ -18,6 +16,7 @@ use deno_core::RcRef; use deno_core::Resource; use deno_core::ResourceId; use futures::future::poll_fn; +use futures::TryFutureExt; use std::borrow::Cow; use std::cell::RefCell; use std::cell::RefMut; @@ -31,6 +30,14 @@ use std::task::Context; use std::task::Poll; use std::task::Waker; +#[derive(Debug, thiserror::Error)] +pub enum StreamResourceError { + #[error(transparent)] + Canceled(#[from] deno_core::Canceled), + #[error("{0}")] + Js(String), +} + // How many buffers we'll allow in the channel before we stop allowing writes. const BUFFER_CHANNEL_SIZE: u16 = 1024; @@ -48,7 +55,7 @@ struct BoundedBufferChannelInner { buffers: [MaybeUninit>; BUFFER_CHANNEL_SIZE as _], ring_producer: u16, ring_consumer: u16, - error: Option, + error: Option, current_size: usize, // TODO(mmastrac): we can math this field instead of accounting for it len: usize, @@ -141,7 +148,10 @@ impl BoundedBufferChannelInner { self.len = 0; } - pub fn read(&mut self, limit: usize) -> Result, AnyError> { + pub fn read( + &mut self, + limit: usize, + ) -> Result, StreamResourceError> { // Empty buffers will return the error, if one exists, or None if self.len == 0 { if let Some(error) = self.error.take() { @@ -230,7 +240,7 @@ impl BoundedBufferChannelInner { Ok(()) } - pub fn write_error(&mut self, error: AnyError) { + pub fn write_error(&mut self, error: StreamResourceError) { self.error = Some(error); if let Some(waker) = self.read_waker.take() { waker.wake(); @@ -306,7 +316,10 @@ impl BoundedBufferChannel { self.inner.borrow_mut() } - pub fn read(&self, limit: usize) -> Result, AnyError> { + pub fn read( + &self, + limit: usize, + ) -> Result, StreamResourceError> { self.inner().read(limit) } @@ -314,7 +327,7 @@ impl BoundedBufferChannel { self.inner().write(buffer) } - pub fn write_error(&self, error: AnyError) { + pub fn write_error(&self, error: StreamResourceError) { self.inner().write_error(error) } @@ -358,7 +371,10 @@ impl ReadableStreamResource { RcRef::map(self, |s| &s.cancel_handle).clone() } - async fn read(self: Rc, limit: usize) -> Result { + async fn read( + self: Rc, + limit: usize, + ) -> Result { let cancel_handle = self.cancel_handle(); // Serialize all the reads using a task queue. let _read_permit = self.read_queue.acquire().await; @@ -387,7 +403,7 @@ impl Resource for ReadableStreamResource { } fn read(self: Rc, limit: usize) -> AsyncResult { - Box::pin(ReadableStreamResource::read(self, limit)) + Box::pin(ReadableStreamResource::read(self, limit).map_err(|e| e.into())) } fn close(self: Rc) { @@ -550,7 +566,7 @@ pub fn op_readable_stream_resource_write_error( ) -> bool { let sender = get_sender(sender); // We can always write an error, no polling required - sender.write_error(type_error(Cow::Owned(error))); + sender.write_error(StreamResourceError::Js(error)); !sender.closed() } diff --git a/ext/web/timers.rs b/ext/web/timers.rs index a9ab7c97e4bdbc..06444ed34ffba3 100644 --- a/ext/web/timers.rs +++ b/ext/web/timers.rs @@ -4,7 +4,10 @@ use deno_core::op2; use deno_core::OpState; +use std::time::Duration; use std::time::Instant; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; pub trait TimersPermission { fn allow_hrtime(&mut self) -> bool; @@ -17,21 +20,28 @@ impl TimersPermission for deno_permissions::PermissionsContainer { } } -pub type StartTime = Instant; +pub struct StartTime(Instant); -// Returns a milliseconds and nanoseconds subsec -// since the start time of the deno runtime. -// If the High precision flag is not set, the -// nanoseconds are rounded on 2ms. -#[op2(fast)] -pub fn op_now(state: &mut OpState, #[buffer] buf: &mut [u8]) +impl Default for StartTime { + fn default() -> Self { + Self(Instant::now()) + } +} + +impl std::ops::Deref for StartTime { + type Target = Instant; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +fn expose_time(state: &mut OpState, duration: Duration, out: &mut [u8]) where TP: TimersPermission + 'static, { - let start_time = state.borrow::(); - let elapsed = start_time.elapsed(); - let seconds = elapsed.as_secs(); - let mut subsec_nanos = elapsed.subsec_nanos(); + let seconds = duration.as_secs() as u32; + let mut subsec_nanos = duration.subsec_nanos(); // If the permission is not enabled // Round the nano result on 2 milliseconds @@ -40,14 +50,33 @@ where let reduced_time_precision = 2_000_000; // 2ms in nanoseconds subsec_nanos -= subsec_nanos % reduced_time_precision; } - if buf.len() < 8 { - return; + + if out.len() >= 8 { + out[0..4].copy_from_slice(&seconds.to_ne_bytes()); + out[4..8].copy_from_slice(&subsec_nanos.to_ne_bytes()); } - let buf: &mut [u32] = - // SAFETY: buffer is at least 8 bytes long. - unsafe { std::slice::from_raw_parts_mut(buf.as_mut_ptr() as _, 2) }; - buf[0] = seconds as u32; - buf[1] = subsec_nanos; +} + +#[op2(fast)] +pub fn op_now(state: &mut OpState, #[buffer] buf: &mut [u8]) +where + TP: TimersPermission + 'static, +{ + let start_time = state.borrow::(); + let elapsed = start_time.elapsed(); + expose_time::(state, elapsed, buf); +} + +#[op2(fast)] +pub fn op_time_origin(state: &mut OpState, #[buffer] buf: &mut [u8]) +where + TP: TimersPermission + 'static, +{ + // https://w3c.github.io/hr-time/#dfn-estimated-monotonic-time-of-the-unix-epoch + let wall_time = SystemTime::now(); + let monotonic_time = state.borrow::().elapsed(); + let epoch = wall_time.duration_since(UNIX_EPOCH).unwrap() - monotonic_time; + expose_time::(state, epoch, buf); } #[allow(clippy::unused_async)] diff --git a/ext/webgpu/01_webgpu.js b/ext/webgpu/01_webgpu.js index 719877750b8a6a..d371f49ea10530 100644 --- a/ext/webgpu/01_webgpu.js +++ b/ext/webgpu/01_webgpu.js @@ -98,6 +98,11 @@ const { ArrayPrototypePush, DataViewPrototypeGetBuffer, Error, + Number, + NumberPOSITIVE_INFINITY, + NumberMAX_SAFE_INTEGER, + NumberNEGATIVE_INFINITY, + NumberMIN_SAFE_INTEGER, MathMax, ObjectDefineProperty, ObjectHasOwn, @@ -614,6 +619,19 @@ function createGPUSupportedLimits(limits) { return adapterFeatures; } +function normalizeLimit(limit) { + if (typeof limit === "bigint") { + limit = Number(limit); + if (limit === NumberPOSITIVE_INFINITY) { + limit = NumberMAX_SAFE_INTEGER; + } else if (limit === NumberNEGATIVE_INFINITY) { + limit = NumberMIN_SAFE_INTEGER; + } + } + + return limit; +} + /** * @typedef InnerAdapterLimits * @property {number} maxTextureDimension1D @@ -653,123 +671,127 @@ class GPUSupportedLimits { get maxTextureDimension1D() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxTextureDimension1D; + return normalizeLimit(this[_limits].maxTextureDimension1D); } get maxTextureDimension2D() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxTextureDimension2D; + return normalizeLimit(this[_limits].maxTextureDimension2D); } get maxTextureDimension3D() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxTextureDimension3D; + return normalizeLimit(this[_limits].maxTextureDimension3D); } get maxTextureArrayLayers() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxTextureArrayLayers; + return normalizeLimit(this[_limits].maxTextureArrayLayers); } get maxBindGroups() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxBindGroups; + return normalizeLimit(this[_limits].maxBindGroups); } get maxBindingsPerBindGroup() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxBindingsPerBindGroup; + return normalizeLimit(this[_limits].maxBindingsPerBindGroup); } get maxBufferSize() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxBufferSize; + return normalizeLimit(this[_limits].maxBufferSize); } get maxDynamicUniformBuffersPerPipelineLayout() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxDynamicUniformBuffersPerPipelineLayout; + return normalizeLimit( + this[_limits].maxDynamicUniformBuffersPerPipelineLayout, + ); } get maxDynamicStorageBuffersPerPipelineLayout() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxDynamicStorageBuffersPerPipelineLayout; + return normalizeLimit( + this[_limits].maxDynamicStorageBuffersPerPipelineLayout, + ); } get maxSampledTexturesPerShaderStage() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxSampledTexturesPerShaderStage; + return normalizeLimit(this[_limits].maxSampledTexturesPerShaderStage); } get maxSamplersPerShaderStage() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxSamplersPerShaderStage; + return normalizeLimit(this[_limits].maxSamplersPerShaderStage); } get maxStorageBuffersPerShaderStage() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxStorageBuffersPerShaderStage; + return normalizeLimit(this[_limits].maxStorageBuffersPerShaderStage); } get maxStorageTexturesPerShaderStage() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxStorageTexturesPerShaderStage; + return normalizeLimit(this[_limits].maxStorageTexturesPerShaderStage); } get maxUniformBuffersPerShaderStage() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxUniformBuffersPerShaderStage; + return normalizeLimit(this[_limits].maxUniformBuffersPerShaderStage); } get maxUniformBufferBindingSize() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxUniformBufferBindingSize; + return normalizeLimit(this[_limits].maxUniformBufferBindingSize); } get maxStorageBufferBindingSize() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxStorageBufferBindingSize; + return normalizeLimit(this[_limits].maxStorageBufferBindingSize); } get minUniformBufferOffsetAlignment() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].minUniformBufferOffsetAlignment; + return normalizeLimit(this[_limits].minUniformBufferOffsetAlignment); } get minStorageBufferOffsetAlignment() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].minStorageBufferOffsetAlignment; + return normalizeLimit(this[_limits].minStorageBufferOffsetAlignment); } get maxVertexBuffers() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxVertexBuffers; + return normalizeLimit(this[_limits].maxVertexBuffers); } get maxVertexAttributes() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxVertexAttributes; + return normalizeLimit(this[_limits].maxVertexAttributes); } get maxVertexBufferArrayStride() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxVertexBufferArrayStride; + return normalizeLimit(this[_limits].maxVertexBufferArrayStride); } get maxInterStageShaderComponents() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxInterStageShaderComponents; + return normalizeLimit(this[_limits].maxInterStageShaderComponents); } get maxColorAttachments() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxColorAttachments; + return normalizeLimit(this[_limits].maxColorAttachments); } get maxColorAttachmentBytesPerSample() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxColorAttachmentBytesPerSample; + return normalizeLimit(this[_limits].maxColorAttachmentBytesPerSample); } get maxComputeWorkgroupStorageSize() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxComputeWorkgroupStorageSize; + return normalizeLimit(this[_limits].maxComputeWorkgroupStorageSize); } get maxComputeInvocationsPerWorkgroup() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxComputeInvocationsPerWorkgroup; + return normalizeLimit(this[_limits].maxComputeInvocationsPerWorkgroup); } get maxComputeWorkgroupSizeX() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxComputeWorkgroupSizeX; + return normalizeLimit(this[_limits].maxComputeWorkgroupSizeX); } get maxComputeWorkgroupSizeY() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxComputeWorkgroupSizeY; + return normalizeLimit(this[_limits].maxComputeWorkgroupSizeY); } get maxComputeWorkgroupSizeZ() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxComputeWorkgroupSizeZ; + return normalizeLimit(this[_limits].maxComputeWorkgroupSizeZ); } get maxComputeWorkgroupsPerDimension() { webidl.assertBranded(this, GPUSupportedLimitsPrototype); - return this[_limits].maxComputeWorkgroupsPerDimension; + return normalizeLimit(this[_limits].maxComputeWorkgroupsPerDimension); } [SymbolFor("Deno.privateCustomInspect")](inspect, inspectOptions) { @@ -6982,6 +7004,12 @@ webidl.converters.GPUComputePassEncoder = webidl.createInterfaceConverter( GPUComputePassEncoder.prototype, ); +// INTERFACE: GPUQuerySet +webidl.converters.GPUQuerySet = webidl.createInterfaceConverter( + "GPUQuerySet", + GPUQuerySet.prototype, +); + // DICTIONARY: GPUComputePassTimestampWrites webidl.converters["GPUComputePassTimestampWrites"] = webidl .createDictionaryConverter( @@ -7154,12 +7182,6 @@ webidl.converters["GPURenderPassDepthStencilAttachment"] = webidl dictMembersGPURenderPassDepthStencilAttachment, ); -// INTERFACE: GPUQuerySet -webidl.converters.GPUQuerySet = webidl.createInterfaceConverter( - "GPUQuerySet", - GPUQuerySet.prototype, -); - // DICTIONARY: GPURenderPassTimestampWrites webidl.converters["GPURenderPassTimestampWrites"] = webidl .createDictionaryConverter( diff --git a/ext/webgpu/Cargo.toml b/ext/webgpu/Cargo.toml index b5cff5985519a8..f84550553eae3f 100644 --- a/ext/webgpu/Cargo.toml +++ b/ext/webgpu/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_webgpu" -version = "0.135.0" +version = "0.147.0" authors = ["the Deno authors"] edition.workspace = true license = "MIT" @@ -25,6 +25,7 @@ serde = { workspace = true, features = ["derive"] } tokio = { workspace = true, features = ["full"] } wgpu-types = { workspace = true, features = ["serde"] } raw-window-handle = { workspace = true } +thiserror.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies.wgpu-core] workspace = true diff --git a/ext/webgpu/buffer.rs b/ext/webgpu/buffer.rs index c6cd6f0a74b2df..c2b53890e0ccef 100644 --- a/ext/webgpu/buffer.rs +++ b/ext/webgpu/buffer.rs @@ -1,7 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; use deno_core::Resource; @@ -13,9 +11,18 @@ use std::sync::Arc; use std::sync::Mutex; use std::time::Duration; -use super::error::DomExceptionOperationError; use super::error::WebGpuResult; +#[derive(Debug, thiserror::Error)] +pub enum BufferError { + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error("usage is not valid")] + InvalidUsage, + #[error(transparent)] + Access(wgpu_core::resource::BufferAccessError), +} + pub(crate) struct WebGpuBuffer( pub(crate) super::Instance, pub(crate) wgpu_core::id::BufferId, @@ -46,18 +53,19 @@ pub fn op_webgpu_create_buffer( #[number] size: u64, usage: u32, mapped_at_creation: bool, -) -> Result { +) -> Result { let instance = state.borrow::(); let device_resource = state .resource_table - .get::(device_rid)?; + .get::(device_rid) + .map_err(BufferError::Resource)?; let device = device_resource.1; let descriptor = wgpu_core::resource::BufferDescriptor { label: Some(label), size, usage: wgpu_types::BufferUsages::from_bits(usage) - .ok_or_else(|| type_error("usage is not valid"))?, + .ok_or(BufferError::InvalidUsage)?, mapped_at_creation, }; @@ -77,18 +85,21 @@ pub async fn op_webgpu_buffer_get_map_async( mode: u32, #[number] offset: u64, #[number] size: u64, -) -> Result { +) -> Result { let device; let done = Arc::new(Mutex::new(None)); { let state_ = state.borrow(); let instance = state_.borrow::(); - let buffer_resource = - state_.resource_table.get::(buffer_rid)?; + let buffer_resource = state_ + .resource_table + .get::(buffer_rid) + .map_err(BufferError::Resource)?; let buffer = buffer_resource.1; let device_resource = state_ .resource_table - .get::(device_rid)?; + .get::(device_rid) + .map_err(BufferError::Resource)?; device = device_resource.1; let done_ = done.clone(); @@ -120,9 +131,7 @@ pub async fn op_webgpu_buffer_get_map_async( let result = done.lock().unwrap().take(); match result { Some(Ok(())) => return Ok(WebGpuResult::empty()), - Some(Err(e)) => { - return Err(DomExceptionOperationError::new(&e.to_string()).into()) - } + Some(Err(e)) => return Err(BufferError::Access(e)), None => { { let state = state.borrow(); @@ -143,9 +152,12 @@ pub fn op_webgpu_buffer_get_mapped_range( #[number] offset: u64, #[number] size: Option, #[buffer] buf: &mut [u8], -) -> Result { +) -> Result { let instance = state.borrow::(); - let buffer_resource = state.resource_table.get::(buffer_rid)?; + let buffer_resource = state + .resource_table + .get::(buffer_rid) + .map_err(BufferError::Resource)?; let buffer = buffer_resource.1; let (slice_pointer, range_size) = @@ -154,7 +166,7 @@ pub fn op_webgpu_buffer_get_mapped_range( offset, size )) - .map_err(|e| DomExceptionOperationError::new(&e.to_string()))?; + .map_err(BufferError::Access)?; // SAFETY: guarantee to be safe from wgpu let slice = unsafe { @@ -176,12 +188,16 @@ pub fn op_webgpu_buffer_unmap( #[smi] buffer_rid: ResourceId, #[smi] mapped_rid: ResourceId, #[buffer] buf: Option<&[u8]>, -) -> Result { +) -> Result { let mapped_resource = state .resource_table - .take::(mapped_rid)?; + .take::(mapped_rid) + .map_err(BufferError::Resource)?; let instance = state.borrow::(); - let buffer_resource = state.resource_table.get::(buffer_rid)?; + let buffer_resource = state + .resource_table + .get::(buffer_rid) + .map_err(BufferError::Resource)?; let buffer = buffer_resource.1; if let Some(buf) = buf { diff --git a/ext/webgpu/bundle.rs b/ext/webgpu/bundle.rs index 57158271cce992..d9a5b29539aa93 100644 --- a/ext/webgpu/bundle.rs +++ b/ext/webgpu/bundle.rs @@ -1,7 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; use deno_core::Resource; @@ -13,6 +11,14 @@ use std::rc::Rc; use super::error::WebGpuResult; +#[derive(Debug, thiserror::Error)] +pub enum BundleError { + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error("size must be larger than 0")] + InvalidSize, +} + struct WebGpuRenderBundleEncoder( RefCell, ); @@ -53,7 +59,7 @@ pub struct CreateRenderBundleEncoderArgs { pub fn op_webgpu_create_render_bundle_encoder( state: &mut OpState, #[serde] args: CreateRenderBundleEncoderArgs, -) -> Result { +) -> Result { let device_resource = state .resource_table .get::(args.device_rid)?; @@ -100,7 +106,7 @@ pub fn op_webgpu_render_bundle_encoder_finish( state: &mut OpState, #[smi] render_bundle_encoder_rid: ResourceId, #[string] label: Cow, -) -> Result { +) -> Result { let render_bundle_encoder_resource = state .resource_table @@ -131,7 +137,7 @@ pub fn op_webgpu_render_bundle_encoder_set_bind_group( #[buffer] dynamic_offsets_data: &[u32], #[number] dynamic_offsets_data_start: usize, #[number] dynamic_offsets_data_length: usize, -) -> Result { +) -> Result { let bind_group_resource = state .resource_table @@ -171,7 +177,7 @@ pub fn op_webgpu_render_bundle_encoder_push_debug_group( state: &mut OpState, #[smi] render_bundle_encoder_rid: ResourceId, #[string] group_label: &str, -) -> Result { +) -> Result { let render_bundle_encoder_resource = state .resource_table @@ -195,7 +201,7 @@ pub fn op_webgpu_render_bundle_encoder_push_debug_group( pub fn op_webgpu_render_bundle_encoder_pop_debug_group( state: &mut OpState, #[smi] render_bundle_encoder_rid: ResourceId, -) -> Result { +) -> Result { let render_bundle_encoder_resource = state .resource_table @@ -214,7 +220,7 @@ pub fn op_webgpu_render_bundle_encoder_insert_debug_marker( state: &mut OpState, #[smi] render_bundle_encoder_rid: ResourceId, #[string] marker_label: &str, -) -> Result { +) -> Result { let render_bundle_encoder_resource = state .resource_table @@ -239,7 +245,7 @@ pub fn op_webgpu_render_bundle_encoder_set_pipeline( state: &mut OpState, #[smi] render_bundle_encoder_rid: ResourceId, #[smi] pipeline: ResourceId, -) -> Result { +) -> Result { let render_pipeline_resource = state .resource_table @@ -266,18 +272,17 @@ pub fn op_webgpu_render_bundle_encoder_set_index_buffer( #[serde] index_format: wgpu_types::IndexFormat, #[number] offset: u64, #[number] size: u64, -) -> Result { +) -> Result { let buffer_resource = state .resource_table - .get::(buffer)?; - let render_bundle_encoder_resource = - state - .resource_table - .get::(render_bundle_encoder_rid)?; - let size = Some( - std::num::NonZeroU64::new(size) - .ok_or_else(|| type_error("size must be larger than 0"))?, - ); + .get::(buffer) + .map_err(BundleError::Resource)?; + let render_bundle_encoder_resource = state + .resource_table + .get::(render_bundle_encoder_rid) + .map_err(BundleError::Resource)?; + let size = + Some(std::num::NonZeroU64::new(size).ok_or(BundleError::InvalidSize)?); render_bundle_encoder_resource .0 @@ -296,19 +301,17 @@ pub fn op_webgpu_render_bundle_encoder_set_vertex_buffer( #[smi] buffer: ResourceId, #[number] offset: u64, #[number] size: Option, -) -> Result { +) -> Result { let buffer_resource = state .resource_table - .get::(buffer)?; - let render_bundle_encoder_resource = - state - .resource_table - .get::(render_bundle_encoder_rid)?; + .get::(buffer) + .map_err(BundleError::Resource)?; + let render_bundle_encoder_resource = state + .resource_table + .get::(render_bundle_encoder_rid) + .map_err(BundleError::Resource)?; let size = if let Some(size) = size { - Some( - std::num::NonZeroU64::new(size) - .ok_or_else(|| type_error("size must be larger than 0"))?, - ) + Some(std::num::NonZeroU64::new(size).ok_or(BundleError::InvalidSize)?) } else { None }; @@ -333,7 +336,7 @@ pub fn op_webgpu_render_bundle_encoder_draw( instance_count: u32, first_vertex: u32, first_instance: u32, -) -> Result { +) -> Result { let render_bundle_encoder_resource = state .resource_table @@ -360,7 +363,7 @@ pub fn op_webgpu_render_bundle_encoder_draw_indexed( first_index: u32, base_vertex: i32, first_instance: u32, -) -> Result { +) -> Result { let render_bundle_encoder_resource = state .resource_table @@ -385,7 +388,7 @@ pub fn op_webgpu_render_bundle_encoder_draw_indirect( #[smi] render_bundle_encoder_rid: ResourceId, #[smi] indirect_buffer: ResourceId, #[number] indirect_offset: u64, -) -> Result { +) -> Result { let buffer_resource = state .resource_table .get::(indirect_buffer)?; diff --git a/ext/webgpu/byow.rs b/ext/webgpu/byow.rs index 3a43f416e57b5e..c9e1177b1efe53 100644 --- a/ext/webgpu/byow.rs +++ b/ext/webgpu/byow.rs @@ -1,7 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; use deno_core::ResourceId; @@ -16,6 +14,47 @@ use std::ptr::NonNull; use crate::surface::WebGpuSurface; +#[derive(Debug, thiserror::Error)] +pub enum ByowError { + #[error("Cannot create surface outside of WebGPU context. Did you forget to call `navigator.gpu.requestAdapter()`?")] + WebGPUNotInitiated, + #[error("Invalid parameters")] + InvalidParameters, + #[error(transparent)] + CreateSurface(wgpu_core::instance::CreateSurfaceError), + #[cfg(target_os = "windows")] + #[error("Invalid system on Windows")] + InvalidSystem, + #[cfg(target_os = "macos")] + #[error("Invalid system on macOS")] + InvalidSystem, + #[cfg(any( + target_os = "linux", + target_os = "freebsd", + target_os = "openbsd" + ))] + #[error("Invalid system on Linux/BSD")] + InvalidSystem, + #[cfg(any( + target_os = "windows", + target_os = "linux", + target_os = "freebsd", + target_os = "openbsd" + ))] + #[error("window is null")] + NullWindow, + #[cfg(any( + target_os = "linux", + target_os = "freebsd", + target_os = "openbsd" + ))] + #[error("display is null")] + NullDisplay, + #[cfg(target_os = "macos")] + #[error("ns_view is null")] + NSViewDisplay, +} + #[op2(fast)] #[smi] pub fn op_webgpu_surface_create( @@ -23,10 +62,10 @@ pub fn op_webgpu_surface_create( #[string] system: &str, p1: *const c_void, p2: *const c_void, -) -> Result { - let instance = state.try_borrow::().ok_or_else(|| { - type_error("Cannot create surface outside of WebGPU context. Did you forget to call `navigator.gpu.requestAdapter()`?") - })?; +) -> Result { + let instance = state + .try_borrow::() + .ok_or(ByowError::WebGPUNotInitiated)?; // Security note: // // The `p1` and `p2` parameters are pointers to platform-specific window @@ -41,13 +80,15 @@ pub fn op_webgpu_surface_create( // // - Only FFI can export v8::External to user code. if p1.is_null() { - return Err(type_error("Invalid parameters")); + return Err(ByowError::InvalidParameters); } let (win_handle, display_handle) = raw_window(system, p1, p2)?; // SAFETY: see above comment let surface = unsafe { - instance.instance_create_surface(display_handle, win_handle, None)? + instance + .instance_create_surface(display_handle, win_handle, None) + .map_err(ByowError::CreateSurface)? }; let rid = state @@ -66,15 +107,14 @@ fn raw_window( system: &str, _ns_window: *const c_void, ns_view: *const c_void, -) -> Result { +) -> Result { if system != "cocoa" { - return Err(type_error("Invalid system on macOS")); + return Err(ByowError::InvalidSystem); } let win_handle = raw_window_handle::RawWindowHandle::AppKit( raw_window_handle::AppKitWindowHandle::new( - NonNull::new(ns_view as *mut c_void) - .ok_or(type_error("ns_view is null"))?, + NonNull::new(ns_view as *mut c_void).ok_or(ByowError::NSViewDisplay)?, ), ); @@ -89,16 +129,16 @@ fn raw_window( system: &str, window: *const c_void, hinstance: *const c_void, -) -> Result { +) -> Result { use raw_window_handle::WindowsDisplayHandle; if system != "win32" { - return Err(type_error("Invalid system on Windows")); + return Err(ByowError::InvalidSystem); } let win_handle = { let mut handle = raw_window_handle::Win32WindowHandle::new( std::num::NonZeroIsize::new(window as isize) - .ok_or(type_error("window is null"))?, + .ok_or(ByowError::NullWindow)?, ); handle.hinstance = std::num::NonZeroIsize::new(hinstance as isize); @@ -115,7 +155,7 @@ fn raw_window( system: &str, window: *const c_void, display: *const c_void, -) -> Result { +) -> Result { let (win_handle, display_handle); if system == "x11" { win_handle = raw_window_handle::RawWindowHandle::Xlib( @@ -131,19 +171,17 @@ fn raw_window( } else if system == "wayland" { win_handle = raw_window_handle::RawWindowHandle::Wayland( raw_window_handle::WaylandWindowHandle::new( - NonNull::new(window as *mut c_void) - .ok_or(type_error("window is null"))?, + NonNull::new(window as *mut c_void).ok_or(ByowError::NullWindow)?, ), ); display_handle = raw_window_handle::RawDisplayHandle::Wayland( raw_window_handle::WaylandDisplayHandle::new( - NonNull::new(display as *mut c_void) - .ok_or(type_error("display is null"))?, + NonNull::new(display as *mut c_void).ok_or(ByowError::NullDisplay)?, ), ); } else { - return Err(type_error("Invalid system on Linux/BSD")); + return Err(ByowError::InvalidSystem); } Ok((win_handle, display_handle)) @@ -160,6 +198,6 @@ fn raw_window( _system: &str, _window: *const c_void, _display: *const c_void, -) -> Result { - Err(type_error("Unsupported platform")) +) -> Result { + Err(deno_core::error::type_error("Unsupported platform")) } diff --git a/ext/webgpu/error.rs b/ext/webgpu/error.rs index 5b55d506ad328a..f08f765386ead3 100644 --- a/ext/webgpu/error.rs +++ b/ext/webgpu/error.rs @@ -1,11 +1,9 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::AnyError; use deno_core::ResourceId; use serde::Serialize; use std::convert::From; use std::error::Error; -use std::fmt; use wgpu_core::binding_model::CreateBindGroupError; use wgpu_core::binding_model::CreateBindGroupLayoutError; use wgpu_core::binding_model::CreatePipelineLayoutError; @@ -286,29 +284,3 @@ impl From for WebGpuError { WebGpuError::Validation(fmt_err(&err)) } } - -#[derive(Debug)] -pub struct DomExceptionOperationError { - pub msg: String, -} - -impl DomExceptionOperationError { - pub fn new(msg: &str) -> Self { - DomExceptionOperationError { - msg: msg.to_string(), - } - } -} - -impl fmt::Display for DomExceptionOperationError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.pad(&self.msg) - } -} - -impl std::error::Error for DomExceptionOperationError {} - -pub fn get_error_class_name(e: &AnyError) -> Option<&'static str> { - e.downcast_ref::() - .map(|_| "DOMExceptionOperationError") -} diff --git a/ext/webgpu/lib.rs b/ext/webgpu/lib.rs index df2ab323af2115..5dc8278e410b54 100644 --- a/ext/webgpu/lib.rs +++ b/ext/webgpu/lib.rs @@ -2,7 +2,6 @@ #![cfg(not(target_arch = "wasm32"))] #![warn(unsafe_op_in_unsafe_fn)] -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; use deno_core::Resource; @@ -16,7 +15,6 @@ use std::rc::Rc; pub use wgpu_core; pub use wgpu_types; -use error::DomExceptionOperationError; use error::WebGpuResult; pub const UNSTABLE_FEATURE_NAME: &str = "webgpu"; @@ -44,7 +42,7 @@ mod macros { #[cfg(all(not(target_arch = "wasm32"), windows))] wgpu_types::Backend::Dx12 => $($c)*.$method:: $params, #[cfg(any( - all(unix, not(target_os = "macos"), not(target_os = "ios")), + all(not(target_os = "macos"), not(target_os = "ios")), feature = "angle", target_arch = "wasm32" ))] @@ -85,6 +83,18 @@ pub mod shader; pub mod surface; pub mod texture; +#[derive(Debug, thiserror::Error)] +pub enum InitError { + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error(transparent)] + InvalidAdapter(wgpu_core::instance::InvalidAdapter), + #[error(transparent)] + RequestDevice(wgpu_core::instance::RequestDeviceError), + #[error(transparent)] + InvalidDevice(wgpu_core::device::InvalidDevice), +} + pub type Instance = std::sync::Arc; struct WebGpuAdapter(Instance, wgpu_core::id::AdapterId); @@ -400,7 +410,7 @@ pub fn op_webgpu_request_adapter( state: Rc>, #[serde] power_preference: Option, force_fallback_adapter: bool, -) -> Result { +) -> Result { let mut state = state.borrow_mut(); let backends = std::env::var("DENO_WEBGPU_BACKEND").map_or_else( @@ -441,10 +451,11 @@ pub fn op_webgpu_request_adapter( } }; let adapter_features = - gfx_select!(adapter => instance.adapter_features(adapter))?; + gfx_select!(adapter => instance.adapter_features(adapter)) + .map_err(InitError::InvalidAdapter)?; let features = deserialize_features(&adapter_features); - let adapter_limits = - gfx_select!(adapter => instance.adapter_limits(adapter))?; + let adapter_limits = gfx_select!(adapter => instance.adapter_limits(adapter)) + .map_err(InitError::InvalidAdapter)?; let instance = instance.clone(); @@ -663,10 +674,12 @@ pub fn op_webgpu_request_device( #[string] label: String, #[serde] required_features: GpuRequiredFeatures, #[serde] required_limits: Option, -) -> Result { +) -> Result { let mut state = state.borrow_mut(); - let adapter_resource = - state.resource_table.take::(adapter_rid)?; + let adapter_resource = state + .resource_table + .take::(adapter_rid) + .map_err(InitError::Resource)?; let adapter = adapter_resource.1; let instance = state.borrow::(); @@ -685,13 +698,14 @@ pub fn op_webgpu_request_device( )); adapter_resource.close(); if let Some(err) = maybe_err { - return Err(DomExceptionOperationError::new(&err.to_string()).into()); + return Err(InitError::RequestDevice(err)); } - let device_features = - gfx_select!(device => instance.device_features(device))?; + let device_features = gfx_select!(device => instance.device_features(device)) + .map_err(InitError::InvalidDevice)?; let features = deserialize_features(&device_features); - let limits = gfx_select!(device => instance.device_limits(device))?; + let limits = gfx_select!(device => instance.device_limits(device)) + .map_err(InitError::InvalidDevice)?; let instance = instance.clone(); let instance2 = instance.clone(); @@ -722,14 +736,17 @@ pub struct GPUAdapterInfo { pub fn op_webgpu_request_adapter_info( state: Rc>, #[smi] adapter_rid: ResourceId, -) -> Result { +) -> Result { let state = state.borrow_mut(); - let adapter_resource = - state.resource_table.get::(adapter_rid)?; + let adapter_resource = state + .resource_table + .get::(adapter_rid) + .map_err(InitError::Resource)?; let adapter = adapter_resource.1; let instance = state.borrow::(); - let info = gfx_select!(adapter => instance.adapter_get_info(adapter))?; + let info = gfx_select!(adapter => instance.adapter_get_info(adapter)) + .map_err(InitError::InvalidAdapter)?; Ok(GPUAdapterInfo { vendor: info.vendor.to_string(), @@ -770,9 +787,11 @@ impl From for wgpu_types::QueryType { pub fn op_webgpu_create_query_set( state: &mut OpState, #[serde] args: CreateQuerySetArgs, -) -> Result { - let device_resource = - state.resource_table.get::(args.device_rid)?; +) -> Result { + let device_resource = state + .resource_table + .get::(args.device_rid) + .map_err(InitError::Resource)?; let device = device_resource.1; let instance = state.borrow::(); diff --git a/ext/webgpu/render_pass.rs b/ext/webgpu/render_pass.rs index c68be3d99a1511..9b9d87d9fc05a6 100644 --- a/ext/webgpu/render_pass.rs +++ b/ext/webgpu/render_pass.rs @@ -1,7 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; use deno_core::Resource; @@ -12,6 +10,14 @@ use std::cell::RefCell; use super::error::WebGpuResult; +#[derive(Debug, thiserror::Error)] +pub enum RenderPassError { + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error("size must be larger than 0")] + InvalidSize, +} + pub(crate) struct WebGpuRenderPass( pub(crate) RefCell, ); @@ -38,7 +44,7 @@ pub struct RenderPassSetViewportArgs { pub fn op_webgpu_render_pass_set_viewport( state: &mut OpState, #[serde] args: RenderPassSetViewportArgs, -) -> Result { +) -> Result { let render_pass_resource = state .resource_table .get::(args.render_pass_rid)?; @@ -65,7 +71,7 @@ pub fn op_webgpu_render_pass_set_scissor_rect( y: u32, width: u32, height: u32, -) -> Result { +) -> Result { let render_pass_resource = state .resource_table .get::(render_pass_rid)?; @@ -87,7 +93,7 @@ pub fn op_webgpu_render_pass_set_blend_constant( state: &mut OpState, #[smi] render_pass_rid: ResourceId, #[serde] color: wgpu_types::Color, -) -> Result { +) -> Result { let render_pass_resource = state .resource_table .get::(render_pass_rid)?; @@ -106,7 +112,7 @@ pub fn op_webgpu_render_pass_set_stencil_reference( state: &mut OpState, #[smi] render_pass_rid: ResourceId, reference: u32, -) -> Result { +) -> Result { let render_pass_resource = state .resource_table .get::(render_pass_rid)?; @@ -125,7 +131,7 @@ pub fn op_webgpu_render_pass_begin_occlusion_query( state: &mut OpState, #[smi] render_pass_rid: ResourceId, query_index: u32, -) -> Result { +) -> Result { let render_pass_resource = state .resource_table .get::(render_pass_rid)?; @@ -143,7 +149,7 @@ pub fn op_webgpu_render_pass_begin_occlusion_query( pub fn op_webgpu_render_pass_end_occlusion_query( state: &mut OpState, #[smi] render_pass_rid: ResourceId, -) -> Result { +) -> Result { let render_pass_resource = state .resource_table .get::(render_pass_rid)?; @@ -161,7 +167,7 @@ pub fn op_webgpu_render_pass_execute_bundles( state: &mut OpState, #[smi] render_pass_rid: ResourceId, #[serde] bundles: Vec, -) -> Result { +) -> Result { let bundles = bundles .iter() .map(|rid| { @@ -171,7 +177,7 @@ pub fn op_webgpu_render_pass_execute_bundles( .get::(*rid)?; Ok(render_bundle_resource.1) }) - .collect::, AnyError>>()?; + .collect::, deno_core::error::AnyError>>()?; let render_pass_resource = state .resource_table @@ -191,7 +197,7 @@ pub fn op_webgpu_render_pass_end( state: &mut OpState, #[smi] command_encoder_rid: ResourceId, #[smi] render_pass_rid: ResourceId, -) -> Result { +) -> Result { let command_encoder_resource = state .resource_table .get::( @@ -217,7 +223,7 @@ pub fn op_webgpu_render_pass_set_bind_group( #[buffer] dynamic_offsets_data: &[u32], #[number] dynamic_offsets_data_start: usize, #[number] dynamic_offsets_data_length: usize, -) -> Result { +) -> Result { let bind_group_resource = state .resource_table @@ -251,7 +257,7 @@ pub fn op_webgpu_render_pass_push_debug_group( state: &mut OpState, #[smi] render_pass_rid: ResourceId, #[string] group_label: &str, -) -> Result { +) -> Result { let render_pass_resource = state .resource_table .get::(render_pass_rid)?; @@ -270,7 +276,7 @@ pub fn op_webgpu_render_pass_push_debug_group( pub fn op_webgpu_render_pass_pop_debug_group( state: &mut OpState, #[smi] render_pass_rid: ResourceId, -) -> Result { +) -> Result { let render_pass_resource = state .resource_table .get::(render_pass_rid)?; @@ -288,7 +294,7 @@ pub fn op_webgpu_render_pass_insert_debug_marker( state: &mut OpState, #[smi] render_pass_rid: ResourceId, #[string] marker_label: &str, -) -> Result { +) -> Result { let render_pass_resource = state .resource_table .get::(render_pass_rid)?; @@ -308,7 +314,7 @@ pub fn op_webgpu_render_pass_set_pipeline( state: &mut OpState, #[smi] render_pass_rid: ResourceId, pipeline: u32, -) -> Result { +) -> Result { let render_pipeline_resource = state .resource_table @@ -334,19 +340,18 @@ pub fn op_webgpu_render_pass_set_index_buffer( #[serde] index_format: wgpu_types::IndexFormat, #[number] offset: u64, #[number] size: Option, -) -> Result { +) -> Result { let buffer_resource = state .resource_table - .get::(buffer)?; + .get::(buffer) + .map_err(RenderPassError::Resource)?; let render_pass_resource = state .resource_table - .get::(render_pass_rid)?; + .get::(render_pass_rid) + .map_err(RenderPassError::Resource)?; let size = if let Some(size) = size { - Some( - std::num::NonZeroU64::new(size) - .ok_or_else(|| type_error("size must be larger than 0"))?, - ) + Some(std::num::NonZeroU64::new(size).ok_or(RenderPassError::InvalidSize)?) } else { None }; @@ -370,19 +375,18 @@ pub fn op_webgpu_render_pass_set_vertex_buffer( buffer: u32, #[number] offset: u64, #[number] size: Option, -) -> Result { +) -> Result { let buffer_resource = state .resource_table - .get::(buffer)?; + .get::(buffer) + .map_err(RenderPassError::Resource)?; let render_pass_resource = state .resource_table - .get::(render_pass_rid)?; + .get::(render_pass_rid) + .map_err(RenderPassError::Resource)?; let size = if let Some(size) = size { - Some( - std::num::NonZeroU64::new(size) - .ok_or_else(|| type_error("size must be larger than 0"))?, - ) + Some(std::num::NonZeroU64::new(size).ok_or(RenderPassError::InvalidSize)?) } else { None }; @@ -407,7 +411,7 @@ pub fn op_webgpu_render_pass_draw( instance_count: u32, first_vertex: u32, first_instance: u32, -) -> Result { +) -> Result { let render_pass_resource = state .resource_table .get::(render_pass_rid)?; @@ -433,7 +437,7 @@ pub fn op_webgpu_render_pass_draw_indexed( first_index: u32, base_vertex: i32, first_instance: u32, -) -> Result { +) -> Result { let render_pass_resource = state .resource_table .get::(render_pass_rid)?; @@ -457,7 +461,7 @@ pub fn op_webgpu_render_pass_draw_indirect( #[smi] render_pass_rid: ResourceId, indirect_buffer: u32, #[number] indirect_offset: u64, -) -> Result { +) -> Result { let buffer_resource = state .resource_table .get::(indirect_buffer)?; @@ -481,7 +485,7 @@ pub fn op_webgpu_render_pass_draw_indexed_indirect( #[smi] render_pass_rid: ResourceId, indirect_buffer: u32, #[number] indirect_offset: u64, -) -> Result { +) -> Result { let buffer_resource = state .resource_table .get::(indirect_buffer)?; diff --git a/ext/webgpu/sampler.rs b/ext/webgpu/sampler.rs index 27c36802e6d3cc..9fc1269ea7b758 100644 --- a/ext/webgpu/sampler.rs +++ b/ext/webgpu/sampler.rs @@ -1,6 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; use deno_core::Resource; @@ -47,7 +46,7 @@ pub struct CreateSamplerArgs { pub fn op_webgpu_create_sampler( state: &mut OpState, #[serde] args: CreateSamplerArgs, -) -> Result { +) -> Result { let instance = state.borrow::(); let device_resource = state .resource_table diff --git a/ext/webgpu/shader.rs b/ext/webgpu/shader.rs index 0b3991c5dea899..4653bd85bf05c6 100644 --- a/ext/webgpu/shader.rs +++ b/ext/webgpu/shader.rs @@ -1,6 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; use deno_core::Resource; @@ -31,7 +30,7 @@ pub fn op_webgpu_create_shader_module( #[smi] device_rid: ResourceId, #[string] label: Cow, #[string] code: Cow, -) -> Result { +) -> Result { let instance = state.borrow::(); let device_resource = state .resource_table diff --git a/ext/webgpu/surface.rs b/ext/webgpu/surface.rs index 1f6d2c87d20ec1..297eaeb00874ae 100644 --- a/ext/webgpu/surface.rs +++ b/ext/webgpu/surface.rs @@ -1,7 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use super::WebGpuResult; -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; use deno_core::Resource; @@ -11,6 +10,16 @@ use std::borrow::Cow; use std::rc::Rc; use wgpu_types::SurfaceStatus; +#[derive(Debug, thiserror::Error)] +pub enum SurfaceError { + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error("Invalid Surface Status")] + InvalidStatus, + #[error(transparent)] + Surface(wgpu_core::present::SurfaceError), +} + pub struct WebGpuSurface(pub crate::Instance, pub wgpu_core::id::SurfaceId); impl Resource for WebGpuSurface { fn name(&self) -> Cow { @@ -41,7 +50,7 @@ pub struct SurfaceConfigureArgs { pub fn op_webgpu_surface_configure( state: &mut OpState, #[serde] args: SurfaceConfigureArgs, -) -> Result { +) -> Result { let instance = state.borrow::(); let device_resource = state .resource_table @@ -75,18 +84,22 @@ pub fn op_webgpu_surface_get_current_texture( state: &mut OpState, #[smi] device_rid: ResourceId, #[smi] surface_rid: ResourceId, -) -> Result { +) -> Result { let instance = state.borrow::(); let device_resource = state .resource_table - .get::(device_rid)?; + .get::(device_rid) + .map_err(SurfaceError::Resource)?; let device = device_resource.1; - let surface_resource = - state.resource_table.get::(surface_rid)?; + let surface_resource = state + .resource_table + .get::(surface_rid) + .map_err(SurfaceError::Resource)?; let surface = surface_resource.1; let output = - gfx_select!(device => instance.surface_get_current_texture(surface, None))?; + gfx_select!(device => instance.surface_get_current_texture(surface, None)) + .map_err(SurfaceError::Surface)?; match output.status { SurfaceStatus::Good | SurfaceStatus::Suboptimal => { @@ -98,7 +111,7 @@ pub fn op_webgpu_surface_get_current_texture( }); Ok(WebGpuResult::rid(rid)) } - _ => Err(AnyError::msg("Invalid Surface Status")), + _ => Err(SurfaceError::InvalidStatus), } } @@ -107,17 +120,21 @@ pub fn op_webgpu_surface_present( state: &mut OpState, #[smi] device_rid: ResourceId, #[smi] surface_rid: ResourceId, -) -> Result<(), AnyError> { +) -> Result<(), SurfaceError> { let instance = state.borrow::(); let device_resource = state .resource_table - .get::(device_rid)?; + .get::(device_rid) + .map_err(SurfaceError::Resource)?; let device = device_resource.1; - let surface_resource = - state.resource_table.get::(surface_rid)?; + let surface_resource = state + .resource_table + .get::(surface_rid) + .map_err(SurfaceError::Resource)?; let surface = surface_resource.1; - let _ = gfx_select!(device => instance.surface_present(surface))?; + let _ = gfx_select!(device => instance.surface_present(surface)) + .map_err(SurfaceError::Surface)?; Ok(()) } diff --git a/ext/webgpu/texture.rs b/ext/webgpu/texture.rs index 44edd1a887c091..f8a5e05a3e2b15 100644 --- a/ext/webgpu/texture.rs +++ b/ext/webgpu/texture.rs @@ -1,6 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; use deno_core::Resource; @@ -62,7 +61,7 @@ pub struct CreateTextureArgs { pub fn op_webgpu_create_texture( state: &mut OpState, #[serde] args: CreateTextureArgs, -) -> Result { +) -> Result { let instance = state.borrow::(); let device_resource = state .resource_table @@ -111,7 +110,7 @@ pub struct CreateTextureViewArgs { pub fn op_webgpu_create_texture_view( state: &mut OpState, #[serde] args: CreateTextureViewArgs, -) -> Result { +) -> Result { let instance = state.borrow::(); let texture_resource = state .resource_table diff --git a/ext/webidl/00_webidl.js b/ext/webidl/00_webidl.js index 1d05aae5fa96ba..eb18cbcc3e4152 100644 --- a/ext/webidl/00_webidl.js +++ b/ext/webidl/00_webidl.js @@ -26,6 +26,7 @@ const { Float32Array, Float64Array, FunctionPrototypeBind, + FunctionPrototypeCall, Int16Array, Int32Array, Int8Array, @@ -77,6 +78,7 @@ const { StringPrototypeToWellFormed, Symbol, SymbolIterator, + SymbolAsyncIterator, SymbolToStringTag, TypedArrayPrototypeGetBuffer, TypedArrayPrototypeGetSymbolToStringTag, @@ -920,6 +922,127 @@ function createSequenceConverter(converter) { }; } +function isAsyncIterable(obj) { + if (obj[SymbolAsyncIterator] === undefined) { + if (obj[SymbolIterator] === undefined) { + return false; + } + } + + return true; +} + +const AsyncIterable = Symbol("[[asyncIterable]]"); + +function createAsyncIterableConverter(converter) { + return function ( + V, + prefix = undefined, + context = undefined, + opts = { __proto__: null }, + ) { + if (type(V) !== "Object") { + throw makeException( + TypeError, + "can not be converted to async iterable.", + prefix, + context, + ); + } + + let isAsync = true; + let method = V[SymbolAsyncIterator]; + if (method === undefined) { + method = V[SymbolIterator]; + + if (method === undefined) { + throw makeException( + TypeError, + "is not iterable.", + prefix, + context, + ); + } + + isAsync = false; + } + + return { + value: V, + [AsyncIterable]: AsyncIterable, + open(context) { + const iter = FunctionPrototypeCall(method, V); + if (type(iter) !== "Object") { + throw new TypeError( + `${context} could not be iterated because iterator method did not return object, but ${ + type(iter) + }.`, + ); + } + + let asyncIterator = iter; + + if (!isAsync) { + asyncIterator = { + // deno-lint-ignore require-await + async next() { + // deno-lint-ignore prefer-primordials + return iter.next(); + }, + }; + } + + return { + async next() { + // deno-lint-ignore prefer-primordials + const iterResult = await asyncIterator.next(); + if (type(iterResult) !== "Object") { + throw TypeError( + `${context} failed to iterate next value because the next() method did not return an object, but ${ + type(iterResult) + }.`, + ); + } + + if (iterResult.done) { + return { done: true }; + } + + const iterValue = converter( + iterResult.value, + `${context} failed to iterate next value`, + `The value returned from the next() method`, + opts, + ); + + return { done: false, value: iterValue }; + }, + async return(reason) { + if (asyncIterator.return === undefined) { + return undefined; + } + + // deno-lint-ignore prefer-primordials + const returnPromiseResult = await asyncIterator.return(reason); + if (type(returnPromiseResult) !== "Object") { + throw TypeError( + `${context} failed to close iterator because the return() method did not return an object, but ${ + type(returnPromiseResult) + }.`, + ); + } + + return undefined; + }, + [SymbolAsyncIterator]() { + return this; + }, + }; + }, + }; + }; +} + function createRecordConverter(keyConverter, valueConverter) { return (V, prefix, context, opts) => { if (type(V) !== "Object") { @@ -1302,9 +1425,11 @@ function setlike(obj, objPrototype, readonly) { export { assertBranded, + AsyncIterable, brand, configureInterface, converters, + createAsyncIterableConverter, createBranded, createDictionaryConverter, createEnumConverter, @@ -1315,6 +1440,7 @@ export { createSequenceConverter, illegalConstructor, invokeCallbackFunction, + isAsyncIterable, makeException, mixinPairIterable, requiredArguments, diff --git a/ext/webidl/Cargo.toml b/ext/webidl/Cargo.toml index 2f87dc2197a5cb..a60374bb8d282a 100644 --- a/ext/webidl/Cargo.toml +++ b/ext/webidl/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_webidl" -version = "0.168.0" +version = "0.180.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/webidl/internal.d.ts b/ext/webidl/internal.d.ts index 1ce45463ecc37e..375d548d3220ae 100644 --- a/ext/webidl/internal.d.ts +++ b/ext/webidl/internal.d.ts @@ -438,6 +438,27 @@ declare module "ext:deno_webidl/00_webidl.js" { opts?: any, ) => T[]; + /** + * Create a converter that converts an async iterable of the inner type. + */ + function createAsyncIterableConverter( + converter: ( + v: V, + prefix?: string, + context?: string, + opts?: any, + ) => T, + ): ( + v: any, + prefix?: string, + context?: string, + opts?: any, + ) => ConvertedAsyncIterable; + + interface ConvertedAsyncIterable extends AsyncIterableIterator { + value: V; + } + /** * Create a converter that converts a Promise of the inner type. */ @@ -559,4 +580,9 @@ declare module "ext:deno_webidl/00_webidl.js" { | "Symbol" | "BigInt" | "Object"; + + /** + * Check whether a value is an async iterable. + */ + function isAsyncIterable(v: any): boolean; } diff --git a/ext/websocket/01_websocket.js b/ext/websocket/01_websocket.js index 58f4773101bc62..78572f5f002fab 100644 --- a/ext/websocket/01_websocket.js +++ b/ext/websocket/01_websocket.js @@ -28,6 +28,7 @@ const { ArrayPrototypePush, ArrayPrototypeShift, ArrayPrototypeSome, + Error, ErrorPrototypeToString, ObjectDefineProperties, ObjectPrototypeIsPrototypeOf, @@ -329,10 +330,14 @@ class WebSocket extends EventTarget { webidl.requiredArguments(arguments.length, 1, prefix); data = webidl.converters.WebSocketSend(data, prefix, "Argument 1"); - if (this[_readyState] !== OPEN) { + if (this[_readyState] === CONNECTING) { throw new DOMException("'readyState' not OPEN", "InvalidStateError"); } + if (this[_readyState] !== OPEN) { + return; + } + if (this[_sendQueue].length === 0) { // Fast path if the send queue is empty, for example when only synchronous // data is being sent. @@ -488,8 +493,11 @@ class WebSocket extends EventTarget { /* error */ this[_readyState] = CLOSED; + const message = op_ws_get_error(rid); + const error = new Error(message); const errorEv = new ErrorEvent("error", { - message: op_ws_get_error(rid), + error, + message, }); this.dispatchEvent(errorEv); diff --git a/ext/websocket/Cargo.toml b/ext/websocket/Cargo.toml index 3fc90254666c32..3807cb8cd5ad1b 100644 --- a/ext/websocket/Cargo.toml +++ b/ext/websocket/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_websocket" -version = "0.173.0" +version = "0.185.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -28,4 +28,5 @@ hyper-util.workspace = true once_cell.workspace = true rustls-tokio-stream.workspace = true serde.workspace = true +thiserror.workspace = true tokio.workspace = true diff --git a/ext/websocket/lib.rs b/ext/websocket/lib.rs index 9e320040b99556..5aef1a7a550646 100644 --- a/ext/websocket/lib.rs +++ b/ext/websocket/lib.rs @@ -1,10 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use crate::stream::WebSocketStream; use bytes::Bytes; -use deno_core::anyhow::bail; -use deno_core::error::invalid_hostname; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::futures::TryFutureExt; use deno_core::op2; use deno_core::unsync::spawn; @@ -43,7 +39,6 @@ use serde::Serialize; use std::borrow::Cow; use std::cell::Cell; use std::cell::RefCell; -use std::fmt; use std::future::Future; use std::num::NonZeroUsize; use std::path::PathBuf; @@ -55,6 +50,7 @@ use tokio::io::ReadHalf; use tokio::io::WriteHalf; use tokio::net::TcpStream; +use deno_permissions::PermissionCheckError; use fastwebsockets::CloseCode; use fastwebsockets::FragmentCollectorRead; use fastwebsockets::Frame; @@ -75,11 +71,33 @@ static USE_WRITEV: Lazy = Lazy::new(|| { false }); +#[derive(Debug, thiserror::Error)] +pub enum WebsocketError { + #[error(transparent)] + Url(url::ParseError), + #[error(transparent)] + Permission(#[from] PermissionCheckError), + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error(transparent)] + Uri(#[from] http::uri::InvalidUri), + #[error("{0}")] + Io(#[from] std::io::Error), + #[error(transparent)] + WebSocket(#[from] fastwebsockets::WebSocketError), + #[error("failed to connect to WebSocket: {0}")] + ConnectionFailed(#[from] HandshakeError), + #[error(transparent)] + Canceled(#[from] deno_core::Canceled), +} + #[derive(Clone)] pub struct WsRootStoreProvider(Option>); impl WsRootStoreProvider { - pub fn get_or_try_init(&self) -> Result, AnyError> { + pub fn get_or_try_init( + &self, + ) -> Result, deno_core::error::AnyError> { Ok(match &self.0 { Some(provider) => Some(provider.get_or_try_init()?.clone()), None => None, @@ -95,7 +113,7 @@ pub trait WebSocketPermissions { &mut self, _url: &url::Url, _api_name: &str, - ) -> Result<(), AnyError>; + ) -> Result<(), PermissionCheckError>; } impl WebSocketPermissions for deno_permissions::PermissionsContainer { @@ -104,7 +122,7 @@ impl WebSocketPermissions for deno_permissions::PermissionsContainer { &mut self, url: &url::Url, api_name: &str, - ) -> Result<(), AnyError> { + ) -> Result<(), PermissionCheckError> { deno_permissions::PermissionsContainer::check_net_url(self, url, api_name) } } @@ -130,20 +148,21 @@ impl Resource for WsCancelResource { // This op is needed because creating a WS instance in JavaScript is a sync // operation and should throw error when permissions are not fulfilled, // but actual op that connects WS is async. -#[op2] +#[op2(stack_trace)] #[smi] pub fn op_ws_check_permission_and_cancel_handle( state: &mut OpState, #[string] api_name: String, #[string] url: String, cancel_handle: bool, -) -> Result, AnyError> +) -> Result, WebsocketError> where WP: WebSocketPermissions + 'static, { - state - .borrow_mut::() - .check_net_url(&url::Url::parse(&url)?, &api_name)?; + state.borrow_mut::().check_net_url( + &url::Url::parse(&url).map_err(WebsocketError::Url)?, + &api_name, + )?; if cancel_handle { let rid = state @@ -163,16 +182,46 @@ pub struct CreateResponse { extensions: String, } +#[derive(Debug, thiserror::Error)] +pub enum HandshakeError { + #[error("Missing path in url")] + MissingPath, + #[error("Invalid status code {0}")] + InvalidStatusCode(StatusCode), + #[error(transparent)] + Http(#[from] http::Error), + #[error(transparent)] + WebSocket(#[from] fastwebsockets::WebSocketError), + #[error("Didn't receive h2 alpn, aborting connection")] + NoH2Alpn, + #[error(transparent)] + Rustls(#[from] deno_tls::rustls::Error), + #[error(transparent)] + Io(#[from] std::io::Error), + #[error(transparent)] + H2(#[from] h2::Error), + #[error("Invalid hostname: '{0}'")] + InvalidHostname(String), + #[error(transparent)] + RootStoreError(deno_core::error::AnyError), + #[error(transparent)] + Tls(deno_tls::TlsError), + #[error(transparent)] + HeaderName(#[from] http::header::InvalidHeaderName), + #[error(transparent)] + HeaderValue(#[from] http::header::InvalidHeaderValue), +} + async fn handshake_websocket( state: &Rc>, uri: &Uri, protocols: &str, headers: Option>, -) -> Result<(WebSocket, http::HeaderMap), AnyError> { +) -> Result<(WebSocket, http::HeaderMap), HandshakeError> { let mut request = Request::builder().method(Method::GET).uri( uri .path_and_query() - .ok_or(type_error("Missing path in url".to_string()))? + .ok_or(HandshakeError::MissingPath)? .as_str(), ); @@ -194,7 +243,9 @@ async fn handshake_websocket( request = populate_common_request_headers(request, &user_agent, protocols, &headers)?; - let request = request.body(http_body_util::Empty::new())?; + let request = request + .body(http_body_util::Empty::new()) + .map_err(HandshakeError::Http)?; let domain = &uri.host().unwrap().to_string(); let port = &uri.port_u16().unwrap_or(match uri.scheme_str() { Some("wss") => 443, @@ -231,7 +282,7 @@ async fn handshake_websocket( async fn handshake_http1_ws( request: Request>, addr: &String, -) -> Result<(WebSocket, http::HeaderMap), AnyError> { +) -> Result<(WebSocket, http::HeaderMap), HandshakeError> { let tcp_socket = TcpStream::connect(addr).await?; handshake_connection(request, tcp_socket).await } @@ -241,11 +292,11 @@ async fn handshake_http1_wss( request: Request>, domain: &str, addr: &str, -) -> Result<(WebSocket, http::HeaderMap), AnyError> { +) -> Result<(WebSocket, http::HeaderMap), HandshakeError> { let tcp_socket = TcpStream::connect(addr).await?; let tls_config = create_ws_client_config(state, SocketUse::Http1Only)?; let dnsname = ServerName::try_from(domain.to_string()) - .map_err(|_| invalid_hostname(domain))?; + .map_err(|_| HandshakeError::InvalidHostname(domain.to_string()))?; let mut tls_connector = TlsStream::new_client_side( tcp_socket, ClientConnection::new(tls_config.into(), dnsname)?, @@ -266,11 +317,11 @@ async fn handshake_http2_wss( domain: &str, headers: &Option>, addr: &str, -) -> Result<(WebSocket, http::HeaderMap), AnyError> { +) -> Result<(WebSocket, http::HeaderMap), HandshakeError> { let tcp_socket = TcpStream::connect(addr).await?; let tls_config = create_ws_client_config(state, SocketUse::Http2Only)?; let dnsname = ServerName::try_from(domain.to_string()) - .map_err(|_| invalid_hostname(domain))?; + .map_err(|_| HandshakeError::InvalidHostname(domain.to_string()))?; // We need to better expose the underlying errors here let mut tls_connector = TlsStream::new_client_side( tcp_socket, @@ -279,7 +330,7 @@ async fn handshake_http2_wss( ); let handshake = tls_connector.handshake().await?; if handshake.alpn.is_none() { - bail!("Didn't receive h2 alpn, aborting connection"); + return Err(HandshakeError::NoH2Alpn); } let h2 = h2::client::Builder::new(); let (mut send, conn) = h2.handshake::<_, Bytes>(tls_connector).await?; @@ -298,7 +349,7 @@ async fn handshake_http2_wss( let (resp, send) = send.send_request(request.body(())?, false)?; let resp = resp.await?; if resp.status() != StatusCode::OK { - bail!("Invalid status code: {}", resp.status()); + return Err(HandshakeError::InvalidStatusCode(resp.status())); } let (http::response::Parts { headers, .. }, recv) = resp.into_parts(); let mut stream = WebSocket::after_handshake( @@ -317,7 +368,7 @@ async fn handshake_connection< >( request: Request>, socket: S, -) -> Result<(WebSocket, http::HeaderMap), AnyError> { +) -> Result<(WebSocket, http::HeaderMap), HandshakeError> { let (upgraded, response) = fastwebsockets::handshake::client(&LocalExecutor, request, socket).await?; @@ -332,7 +383,7 @@ async fn handshake_connection< pub fn create_ws_client_config( state: &Rc>, socket_use: SocketUse, -) -> Result { +) -> Result { let unsafely_ignore_certificate_errors: Option> = state .borrow() .try_borrow::() @@ -340,7 +391,8 @@ pub fn create_ws_client_config( let root_cert_store = state .borrow() .borrow::() - .get_or_try_init()?; + .get_or_try_init() + .map_err(HandshakeError::RootStoreError)?; create_client_config( root_cert_store, @@ -349,6 +401,7 @@ pub fn create_ws_client_config( TlsKeys::Null, socket_use, ) + .map_err(HandshakeError::Tls) } /// Headers common to both http/1.1 and h2 requests. @@ -357,7 +410,7 @@ fn populate_common_request_headers( user_agent: &str, protocols: &str, headers: &Option>, -) -> Result { +) -> Result { request = request .header("User-Agent", user_agent) .header("Sec-WebSocket-Version", "13"); @@ -368,10 +421,8 @@ fn populate_common_request_headers( if let Some(headers) = headers { for (key, value) in headers { - let name = HeaderName::from_bytes(key) - .map_err(|err| type_error(err.to_string()))?; - let v = HeaderValue::from_bytes(value) - .map_err(|err| type_error(err.to_string()))?; + let name = HeaderName::from_bytes(key)?; + let v = HeaderValue::from_bytes(value)?; let is_disallowed_header = matches!( name, @@ -392,7 +443,7 @@ fn populate_common_request_headers( Ok(request) } -#[op2(async)] +#[op2(async, stack_trace)] #[serde] pub async fn op_ws_create( state: Rc>, @@ -401,14 +452,17 @@ pub async fn op_ws_create( #[string] protocols: String, #[smi] cancel_handle: Option, #[serde] headers: Option>, -) -> Result +) -> Result where WP: WebSocketPermissions + 'static, { { let mut s = state.borrow_mut(); s.borrow_mut::() - .check_net_url(&url::Url::parse(&url)?, &api_name) + .check_net_url( + &url::Url::parse(&url).map_err(WebsocketError::Url)?, + &api_name, + ) .expect( "Permission check should have been done in op_ws_check_permission", ); @@ -418,7 +472,8 @@ where let r = state .borrow_mut() .resource_table - .get::(cancel_rid)?; + .get::(cancel_rid) + .map_err(WebsocketError::Resource)?; Some(r.0.clone()) } else { None @@ -427,15 +482,11 @@ where let uri: Uri = url.parse()?; let handshake = handshake_websocket(&state, &uri, &protocols, headers) - .map_err(|err| { - AnyError::from(DomExceptionNetworkError::new(&format!( - "failed to connect to WebSocket: {err}" - ))) - }); + .map_err(WebsocketError::ConnectionFailed); let (stream, response) = match cancel_resource { - Some(rc) => handshake.try_or_cancel(rc).await, - None => handshake.await, - }?; + Some(rc) => handshake.try_or_cancel(rc).await?, + None => handshake.await?, + }; if let Some(cancel_rid) = cancel_handle { if let Ok(res) = state.borrow_mut().resource_table.take_any(cancel_rid) { @@ -520,14 +571,12 @@ impl ServerWebSocket { self: &Rc, lock: AsyncMutFuture>>, frame: Frame<'_>, - ) -> Result<(), AnyError> { + ) -> Result<(), WebsocketError> { let mut ws = lock.await; if ws.is_closed() { return Ok(()); } - ws.write_frame(frame) - .await - .map_err(|err| type_error(err.to_string()))?; + ws.write_frame(frame).await?; Ok(()) } } @@ -542,7 +591,7 @@ pub fn ws_create_server_stream( state: &mut OpState, transport: NetworkStream, read_buf: Bytes, -) -> Result { +) -> ResourceId { let mut ws = WebSocket::after_handshake( WebSocketStream::new( stream::WsStreamKind::Network(transport), @@ -554,8 +603,7 @@ pub fn ws_create_server_stream( ws.set_auto_close(true); ws.set_auto_pong(true); - let rid = state.resource_table.add(ServerWebSocket::new(ws)); - Ok(rid) + state.resource_table.add(ServerWebSocket::new(ws)) } fn send_binary(state: &mut OpState, rid: ResourceId, data: &[u8]) { @@ -625,11 +673,12 @@ pub async fn op_ws_send_binary_async( state: Rc>, #[smi] rid: ResourceId, #[buffer] data: JsBuffer, -) -> Result<(), AnyError> { +) -> Result<(), WebsocketError> { let resource = state .borrow_mut() .resource_table - .get::(rid)?; + .get::(rid) + .map_err(WebsocketError::Resource)?; let data = data.to_vec(); let lock = resource.reserve_lock(); resource @@ -643,11 +692,12 @@ pub async fn op_ws_send_text_async( state: Rc>, #[smi] rid: ResourceId, #[string] data: String, -) -> Result<(), AnyError> { +) -> Result<(), WebsocketError> { let resource = state .borrow_mut() .resource_table - .get::(rid)?; + .get::(rid) + .map_err(WebsocketError::Resource)?; let lock = resource.reserve_lock(); resource .write_frame( @@ -677,11 +727,12 @@ pub fn op_ws_get_buffered_amount( pub async fn op_ws_send_ping( state: Rc>, #[smi] rid: ResourceId, -) -> Result<(), AnyError> { +) -> Result<(), WebsocketError> { let resource = state .borrow_mut() .resource_table - .get::(rid)?; + .get::(rid) + .map_err(WebsocketError::Resource)?; let lock = resource.reserve_lock(); resource .write_frame( @@ -697,7 +748,7 @@ pub async fn op_ws_close( #[smi] rid: ResourceId, #[smi] code: Option, #[string] reason: Option, -) -> Result<(), AnyError> { +) -> Result<(), WebsocketError> { let Ok(resource) = state .borrow_mut() .resource_table @@ -712,8 +763,7 @@ pub async fn op_ws_close( resource.closed.set(true); let lock = resource.reserve_lock(); - resource.write_frame(lock, frame).await?; - Ok(()) + resource.write_frame(lock, frame).await } #[op2] @@ -867,32 +917,6 @@ pub fn get_declaration() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("lib.deno_websocket.d.ts") } -#[derive(Debug)] -pub struct DomExceptionNetworkError { - pub msg: String, -} - -impl DomExceptionNetworkError { - pub fn new(msg: &str) -> Self { - DomExceptionNetworkError { - msg: msg.to_string(), - } - } -} - -impl fmt::Display for DomExceptionNetworkError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.pad(&self.msg) - } -} - -impl std::error::Error for DomExceptionNetworkError {} - -pub fn get_network_error_class_name(e: &AnyError) -> Option<&'static str> { - e.downcast_ref::() - .map(|_| "DOMExceptionNetworkError") -} - // Needed so hyper can use non Send futures #[derive(Clone)] struct LocalExecutor; diff --git a/ext/webstorage/01_webstorage.js b/ext/webstorage/01_webstorage.js index 3cbdd708dd64a9..12abea8387ca3b 100644 --- a/ext/webstorage/01_webstorage.js +++ b/ext/webstorage/01_webstorage.js @@ -3,91 +3,20 @@ /// import { primordials } from "ext:core/mod.js"; -import { - op_webstorage_clear, - op_webstorage_get, - op_webstorage_iterate_keys, - op_webstorage_key, - op_webstorage_length, - op_webstorage_remove, - op_webstorage_set, -} from "ext:core/ops"; +import { op_webstorage_iterate_keys, Storage } from "ext:core/ops"; const { - Symbol, SymbolFor, ObjectFromEntries, ObjectEntries, ReflectDefineProperty, ReflectDeleteProperty, - ReflectGet, + FunctionPrototypeBind, ReflectHas, Proxy, } = primordials; -import * as webidl from "ext:deno_webidl/00_webidl.js"; - -const _persistent = Symbol("[[persistent]]"); - -class Storage { - [_persistent]; - - constructor() { - webidl.illegalConstructor(); - } - - get length() { - webidl.assertBranded(this, StoragePrototype); - return op_webstorage_length(this[_persistent]); - } - - key(index) { - webidl.assertBranded(this, StoragePrototype); - const prefix = "Failed to execute 'key' on 'Storage'"; - webidl.requiredArguments(arguments.length, 1, prefix); - index = webidl.converters["unsigned long"](index, prefix, "Argument 1"); - - return op_webstorage_key(index, this[_persistent]); - } - - setItem(key, value) { - webidl.assertBranded(this, StoragePrototype); - const prefix = "Failed to execute 'setItem' on 'Storage'"; - webidl.requiredArguments(arguments.length, 2, prefix); - key = webidl.converters.DOMString(key, prefix, "Argument 1"); - value = webidl.converters.DOMString(value, prefix, "Argument 2"); - - op_webstorage_set(key, value, this[_persistent]); - } - - getItem(key) { - webidl.assertBranded(this, StoragePrototype); - const prefix = "Failed to execute 'getItem' on 'Storage'"; - webidl.requiredArguments(arguments.length, 1, prefix); - key = webidl.converters.DOMString(key, prefix, "Argument 1"); - - return op_webstorage_get(key, this[_persistent]); - } - - removeItem(key) { - webidl.assertBranded(this, StoragePrototype); - const prefix = "Failed to execute 'removeItem' on 'Storage'"; - webidl.requiredArguments(arguments.length, 1, prefix); - key = webidl.converters.DOMString(key, prefix, "Argument 1"); - - op_webstorage_remove(key, this[_persistent]); - } - - clear() { - webidl.assertBranded(this, StoragePrototype); - op_webstorage_clear(this[_persistent]); - } -} - -const StoragePrototype = Storage.prototype; - function createStorage(persistent) { - const storage = webidl.createBranded(Storage); - storage[_persistent] = persistent; + const storage = new Storage(persistent); const proxy = new Proxy(storage, { deleteProperty(target, key) { @@ -106,12 +35,16 @@ function createStorage(persistent) { return true; }, - get(target, key, receiver) { + get(target, key) { if (typeof key === "symbol") { return target[key]; } if (ReflectHas(target, key)) { - return ReflectGet(target, key, receiver); + const value = target[key]; + if (typeof value === "function") { + return FunctionPrototypeBind(value, target); + } + return value; } return target.getItem(key) ?? undefined; }, @@ -136,13 +69,16 @@ function createStorage(persistent) { }, ownKeys() { - return op_webstorage_iterate_keys(persistent); + return op_webstorage_iterate_keys(storage); }, getOwnPropertyDescriptor(target, key) { if (ReflectHas(target, key)) { return undefined; } + if (typeof key === "symbol") { + return undefined; + } const value = target.getItem(key); if (value === null) { return undefined; @@ -160,7 +96,7 @@ function createStorage(persistent) { inspect, inspectOptions, ) { - return `${this.constructor.name} ${ + return `Storage ${ inspect({ ...ObjectFromEntries(ObjectEntries(proxy)), length: this.length, diff --git a/ext/webstorage/Cargo.toml b/ext/webstorage/Cargo.toml index fbcd17edf9cc55..ca68d176290c2d 100644 --- a/ext/webstorage/Cargo.toml +++ b/ext/webstorage/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_webstorage" -version = "0.163.0" +version = "0.175.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -17,3 +17,4 @@ path = "lib.rs" deno_core.workspace = true deno_web.workspace = true rusqlite.workspace = true +thiserror.workspace = true diff --git a/ext/webstorage/lib.rs b/ext/webstorage/lib.rs index 99e61a1804fad9..c3e4c465969ed3 100644 --- a/ext/webstorage/lib.rs +++ b/ext/webstorage/lib.rs @@ -2,11 +2,10 @@ // NOTE to all: use **cached** prepared statements when interfacing with SQLite. -use std::fmt; use std::path::PathBuf; -use deno_core::error::AnyError; use deno_core::op2; +use deno_core::GarbageCollected; use deno_core::OpState; use rusqlite::params; use rusqlite::Connection; @@ -14,6 +13,18 @@ use rusqlite::OptionalExtension; pub use rusqlite; +#[derive(Debug, thiserror::Error)] +pub enum WebStorageError { + #[error("LocalStorage is not supported in this context.")] + ContextNotSupported, + #[error(transparent)] + Sqlite(#[from] rusqlite::Error), + #[error(transparent)] + Io(std::io::Error), + #[error("Exceeded maximum storage size")] + StorageExceeded, +} + #[derive(Clone)] struct OriginStorageDir(PathBuf); @@ -22,17 +33,14 @@ const MAX_STORAGE_BYTES: usize = 10 * 1024 * 1024; deno_core::extension!(deno_webstorage, deps = [ deno_webidl ], ops = [ - op_webstorage_length, - op_webstorage_key, - op_webstorage_set, - op_webstorage_get, - op_webstorage_remove, - op_webstorage_clear, op_webstorage_iterate_keys, ], + objects = [ + Storage + ], esm = [ "01_webstorage.js" ], options = { - origin_storage_dir: Option + origin_storage_dir: Option }, state = |state, options| { if let Some(origin_storage_dir) = options.origin_storage_dir { @@ -51,15 +59,13 @@ struct SessionStorage(Connection); fn get_webstorage( state: &mut OpState, persistent: bool, -) -> Result<&Connection, AnyError> { +) -> Result<&Connection, WebStorageError> { let conn = if persistent { if state.try_borrow::().is_none() { - let path = state.try_borrow::().ok_or_else(|| { - DomExceptionNotSupportedError::new( - "LocalStorage is not supported in this context.", - ) - })?; - std::fs::create_dir_all(&path.0)?; + let path = state + .try_borrow::() + .ok_or(WebStorageError::ContextNotSupported)?; + std::fs::create_dir_all(&path.0).map_err(WebStorageError::Io)?; let conn = Connection::open(path.0.join("local_storage"))?; // Enable write-ahead-logging and tweak some other stuff. let initial_pragmas = " @@ -102,127 +108,134 @@ fn get_webstorage( Ok(conn) } -#[op2(fast)] -pub fn op_webstorage_length( - state: &mut OpState, - persistent: bool, -) -> Result { - let conn = get_webstorage(state, persistent)?; - - let mut stmt = conn.prepare_cached("SELECT COUNT(*) FROM data")?; - let length: u32 = stmt.query_row(params![], |row| row.get(0))?; +#[inline] +fn size_check(input: usize) -> Result<(), WebStorageError> { + if input >= MAX_STORAGE_BYTES { + return Err(WebStorageError::StorageExceeded); + } - Ok(length) + Ok(()) } -#[op2] -#[string] -pub fn op_webstorage_key( - state: &mut OpState, - #[smi] index: u32, +struct Storage { persistent: bool, -) -> Result, AnyError> { - let conn = get_webstorage(state, persistent)?; +} - let mut stmt = - conn.prepare_cached("SELECT key FROM data LIMIT 1 OFFSET ?")?; +impl GarbageCollected for Storage {} - let key: Option = stmt - .query_row(params![index], |row| row.get(0)) - .optional()?; +#[op2] +impl Storage { + #[constructor] + #[cppgc] + fn new(persistent: bool) -> Storage { + Storage { persistent } + } - Ok(key) -} + #[getter] + #[smi] + fn length(&self, state: &mut OpState) -> Result { + let conn = get_webstorage(state, self.persistent)?; -#[inline] -fn size_check(input: usize) -> Result<(), AnyError> { - if input >= MAX_STORAGE_BYTES { - return Err( - deno_web::DomExceptionQuotaExceededError::new( - "Exceeded maximum storage size", - ) - .into(), - ); + let mut stmt = conn.prepare_cached("SELECT COUNT(*) FROM data")?; + let length: u32 = stmt.query_row(params![], |row| row.get(0))?; + + Ok(length) } - Ok(()) -} + #[required(1)] + #[string] + fn key( + &self, + state: &mut OpState, + #[smi] index: u32, + ) -> Result, WebStorageError> { + let conn = get_webstorage(state, self.persistent)?; -#[op2(fast)] -pub fn op_webstorage_set( - state: &mut OpState, - #[string] key: &str, - #[string] value: &str, - persistent: bool, -) -> Result<(), AnyError> { - let conn = get_webstorage(state, persistent)?; + let mut stmt = + conn.prepare_cached("SELECT key FROM data LIMIT 1 OFFSET ?")?; - size_check(key.len() + value.len())?; + let key: Option = stmt + .query_row(params![index], |row| row.get(0)) + .optional()?; - let mut stmt = conn - .prepare_cached("SELECT SUM(pgsize) FROM dbstat WHERE name = 'data'")?; - let size: u32 = stmt.query_row(params![], |row| row.get(0))?; + Ok(key) + } - size_check(size as usize)?; + #[fast] + #[required(2)] + fn set_item( + &self, + state: &mut OpState, + #[string] key: &str, + #[string] value: &str, + ) -> Result<(), WebStorageError> { + let conn = get_webstorage(state, self.persistent)?; - let mut stmt = conn - .prepare_cached("INSERT OR REPLACE INTO data (key, value) VALUES (?, ?)")?; - stmt.execute(params![key, value])?; + size_check(key.len() + value.len())?; - Ok(()) -} + let mut stmt = conn + .prepare_cached("SELECT SUM(pgsize) FROM dbstat WHERE name = 'data'")?; + let size: u32 = stmt.query_row(params![], |row| row.get(0))?; -#[op2] -#[string] -pub fn op_webstorage_get( - state: &mut OpState, - #[string] key_name: String, - persistent: bool, -) -> Result, AnyError> { - let conn = get_webstorage(state, persistent)?; + size_check(size as usize)?; - let mut stmt = conn.prepare_cached("SELECT value FROM data WHERE key = ?")?; - let val = stmt - .query_row(params![key_name], |row| row.get(0)) - .optional()?; + let mut stmt = conn.prepare_cached( + "INSERT OR REPLACE INTO data (key, value) VALUES (?, ?)", + )?; + stmt.execute(params![key, value])?; - Ok(val) -} + Ok(()) + } -#[op2(fast)] -pub fn op_webstorage_remove( - state: &mut OpState, - #[string] key_name: &str, - persistent: bool, -) -> Result<(), AnyError> { - let conn = get_webstorage(state, persistent)?; + #[required(1)] + #[string] + fn get_item( + &self, + state: &mut OpState, + #[string] key: &str, + ) -> Result, WebStorageError> { + let conn = get_webstorage(state, self.persistent)?; - let mut stmt = conn.prepare_cached("DELETE FROM data WHERE key = ?")?; - stmt.execute(params![key_name])?; + let mut stmt = + conn.prepare_cached("SELECT value FROM data WHERE key = ?")?; + let val = stmt.query_row(params![key], |row| row.get(0)).optional()?; - Ok(()) -} + Ok(val) + } -#[op2(fast)] -pub fn op_webstorage_clear( - state: &mut OpState, - persistent: bool, -) -> Result<(), AnyError> { - let conn = get_webstorage(state, persistent)?; + #[fast] + #[required(1)] + fn remove_item( + &self, + state: &mut OpState, + #[string] key: &str, + ) -> Result<(), WebStorageError> { + let conn = get_webstorage(state, self.persistent)?; - let mut stmt = conn.prepare_cached("DELETE FROM data")?; - stmt.execute(params![])?; + let mut stmt = conn.prepare_cached("DELETE FROM data WHERE key = ?")?; + stmt.execute(params![key])?; - Ok(()) + Ok(()) + } + + #[fast] + fn clear(&self, state: &mut OpState) -> Result<(), WebStorageError> { + let conn = get_webstorage(state, self.persistent)?; + + let mut stmt = conn.prepare_cached("DELETE FROM data")?; + stmt.execute(params![])?; + + Ok(()) + } } #[op2] #[serde] -pub fn op_webstorage_iterate_keys( +fn op_webstorage_iterate_keys( + #[cppgc] storage: &Storage, state: &mut OpState, - persistent: bool, -) -> Result, AnyError> { - let conn = get_webstorage(state, persistent)?; +) -> Result, WebStorageError> { + let conn = get_webstorage(state, storage.persistent)?; let mut stmt = conn.prepare_cached("SELECT key FROM data")?; let keys = stmt @@ -232,31 +245,3 @@ pub fn op_webstorage_iterate_keys( Ok(keys) } - -#[derive(Debug)] -pub struct DomExceptionNotSupportedError { - pub msg: String, -} - -impl DomExceptionNotSupportedError { - pub fn new(msg: &str) -> Self { - DomExceptionNotSupportedError { - msg: msg.to_string(), - } - } -} - -impl fmt::Display for DomExceptionNotSupportedError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.pad(&self.msg) - } -} - -impl std::error::Error for DomExceptionNotSupportedError {} - -pub fn get_not_supported_error_class_name( - e: &AnyError, -) -> Option<&'static str> { - e.downcast_ref::() - .map(|_| "DOMExceptionNotSupportedError") -} diff --git a/resolvers/deno/Cargo.toml b/resolvers/deno/Cargo.toml new file mode 100644 index 00000000000000..f5119478b85795 --- /dev/null +++ b/resolvers/deno/Cargo.toml @@ -0,0 +1,33 @@ +# Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +[package] +name = "deno_resolver" +version = "0.12.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +readme = "README.md" +repository.workspace = true +description = "Deno resolution algorithm" + +[lib] +path = "lib.rs" + +[dependencies] +anyhow.workspace = true +base32.workspace = true +boxed_error.workspace = true +dashmap.workspace = true +deno_config.workspace = true +deno_media_type.workspace = true +deno_package_json.workspace = true +deno_package_json.features = ["sync"] +deno_path_util.workspace = true +deno_semver.workspace = true +node_resolver.workspace = true +node_resolver.features = ["sync"] +thiserror.workspace = true +url.workspace = true + +[dev-dependencies] +test_util.workspace = true diff --git a/resolvers/deno/README.md b/resolvers/deno/README.md new file mode 100644 index 00000000000000..f51619a314e5c5 --- /dev/null +++ b/resolvers/deno/README.md @@ -0,0 +1,3 @@ +# deno_resolver + +Deno resolution algorithm. diff --git a/resolvers/deno/cjs.rs b/resolvers/deno/cjs.rs new file mode 100644 index 00000000000000..9ae60b6a154cb8 --- /dev/null +++ b/resolvers/deno/cjs.rs @@ -0,0 +1,277 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::sync::Arc; + +use dashmap::DashMap; +use deno_media_type::MediaType; +use node_resolver::env::NodeResolverEnv; +use node_resolver::errors::ClosestPkgJsonError; +use node_resolver::InNpmPackageChecker; +use node_resolver::PackageJsonResolver; +use node_resolver::ResolutionMode; +use url::Url; + +/// Keeps track of what module specifiers were resolved as CJS. +/// +/// Modules that are `.js`, `.ts`, `.jsx`, and `tsx` are only known to +/// be CJS or ESM after they're loaded based on their contents. So these +/// files will be "maybe CJS" until they're loaded. +#[derive(Debug)] +pub struct CjsTracker { + is_cjs_resolver: IsCjsResolver, + known: DashMap, +} + +impl CjsTracker { + pub fn new( + in_npm_pkg_checker: Arc, + pkg_json_resolver: Arc>, + mode: IsCjsResolutionMode, + ) -> Self { + Self { + is_cjs_resolver: IsCjsResolver::new( + in_npm_pkg_checker, + pkg_json_resolver, + mode, + ), + known: Default::default(), + } + } + + /// Checks whether the file might be treated as CJS, but it's not for sure + /// yet because the source hasn't been loaded to see whether it contains + /// imports or exports. + pub fn is_maybe_cjs( + &self, + specifier: &Url, + media_type: MediaType, + ) -> Result { + self.treat_as_cjs_with_is_script(specifier, media_type, None) + } + + /// Gets whether the file is CJS. If true, this is for sure + /// cjs because `is_script` is provided. + /// + /// `is_script` should be `true` when the contents of the file at the + /// provided specifier are known to be a script and not an ES module. + pub fn is_cjs_with_known_is_script( + &self, + specifier: &Url, + media_type: MediaType, + is_script: bool, + ) -> Result { + self.treat_as_cjs_with_is_script(specifier, media_type, Some(is_script)) + } + + fn treat_as_cjs_with_is_script( + &self, + specifier: &Url, + media_type: MediaType, + is_script: Option, + ) -> Result { + let kind = match self + .get_known_mode_with_is_script(specifier, media_type, is_script) + { + Some(kind) => kind, + None => self.is_cjs_resolver.check_based_on_pkg_json(specifier)?, + }; + Ok(kind == ResolutionMode::Require) + } + + /// Gets the referrer for the specified module specifier. + /// + /// Generally the referrer should already be tracked by calling + /// `is_cjs_with_known_is_script` before calling this method. + pub fn get_referrer_kind(&self, specifier: &Url) -> ResolutionMode { + if specifier.scheme() != "file" { + return ResolutionMode::Import; + } + self + .get_known_mode(specifier, MediaType::from_specifier(specifier)) + .unwrap_or(ResolutionMode::Import) + } + + fn get_known_mode( + &self, + specifier: &Url, + media_type: MediaType, + ) -> Option { + self.get_known_mode_with_is_script(specifier, media_type, None) + } + + fn get_known_mode_with_is_script( + &self, + specifier: &Url, + media_type: MediaType, + is_script: Option, + ) -> Option { + self.is_cjs_resolver.get_known_mode_with_is_script( + specifier, + media_type, + is_script, + &self.known, + ) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IsCjsResolutionMode { + /// Requires an explicit `"type": "commonjs"` in the package.json. + ExplicitTypeCommonJs, + /// Implicitly uses `"type": "commonjs"` if no `"type"` is specified. + ImplicitTypeCommonJs, + /// Does not respect `"type": "commonjs"` and always treats ambiguous files as ESM. + Disabled, +} + +/// Resolves whether a module is CJS or ESM. +#[derive(Debug)] +pub struct IsCjsResolver { + in_npm_pkg_checker: Arc, + pkg_json_resolver: Arc>, + mode: IsCjsResolutionMode, +} + +impl IsCjsResolver { + pub fn new( + in_npm_pkg_checker: Arc, + pkg_json_resolver: Arc>, + mode: IsCjsResolutionMode, + ) -> Self { + Self { + in_npm_pkg_checker, + pkg_json_resolver, + mode, + } + } + + /// Gets the resolution mode for a module in the LSP. + pub fn get_lsp_resolution_mode( + &self, + specifier: &Url, + is_script: Option, + ) -> ResolutionMode { + if specifier.scheme() != "file" { + return ResolutionMode::Import; + } + match MediaType::from_specifier(specifier) { + MediaType::Mts | MediaType::Mjs | MediaType::Dmts => ResolutionMode::Import, + MediaType::Cjs | MediaType::Cts | MediaType::Dcts => ResolutionMode::Require, + MediaType::Dts => { + // dts files are always determined based on the package.json because + // they contain imports/exports even when considered CJS + self.check_based_on_pkg_json(specifier).unwrap_or(ResolutionMode::Import) + } + MediaType::Wasm | + MediaType::Json => ResolutionMode::Import, + MediaType::JavaScript + | MediaType::Jsx + | MediaType::TypeScript + | MediaType::Tsx + // treat these as unknown + | MediaType::Css + | MediaType::SourceMap + | MediaType::Unknown => { + match is_script { + Some(true) => self.check_based_on_pkg_json(specifier).unwrap_or(ResolutionMode::Import), + Some(false) | None => ResolutionMode::Import, + } + } + } + } + + fn get_known_mode_with_is_script( + &self, + specifier: &Url, + media_type: MediaType, + is_script: Option, + known_cache: &DashMap, + ) -> Option { + if specifier.scheme() != "file" { + return Some(ResolutionMode::Import); + } + + match media_type { + MediaType::Mts | MediaType::Mjs | MediaType::Dmts => Some(ResolutionMode::Import), + MediaType::Cjs | MediaType::Cts | MediaType::Dcts => Some(ResolutionMode::Require), + MediaType::Dts => { + // dts files are always determined based on the package.json because + // they contain imports/exports even when considered CJS + if let Some(value) = known_cache.get(specifier).map(|v| *v) { + Some(value) + } else { + let value = self.check_based_on_pkg_json(specifier).ok(); + if let Some(value) = value { + known_cache.insert(specifier.clone(), value); + } + Some(value.unwrap_or(ResolutionMode::Import)) + } + } + MediaType::Wasm | + MediaType::Json => Some(ResolutionMode::Import), + MediaType::JavaScript + | MediaType::Jsx + | MediaType::TypeScript + | MediaType::Tsx + // treat these as unknown + | MediaType::Css + | MediaType::SourceMap + | MediaType::Unknown => { + if let Some(value) = known_cache.get(specifier).map(|v| *v) { + if value == ResolutionMode::Require && is_script == Some(false) { + // we now know this is actually esm + known_cache.insert(specifier.clone(), ResolutionMode::Import); + Some(ResolutionMode::Import) + } else { + Some(value) + } + } else if is_script == Some(false) { + // we know this is esm + known_cache.insert(specifier.clone(), ResolutionMode::Import); + Some(ResolutionMode::Import) + } else { + None + } + } + } + } + + fn check_based_on_pkg_json( + &self, + specifier: &Url, + ) -> Result { + if self.in_npm_pkg_checker.in_npm_package(specifier) { + if let Some(pkg_json) = + self.pkg_json_resolver.get_closest_package_json(specifier)? + { + let is_file_location_cjs = pkg_json.typ != "module"; + Ok(if is_file_location_cjs { + ResolutionMode::Require + } else { + ResolutionMode::Import + }) + } else { + Ok(ResolutionMode::Require) + } + } else if self.mode != IsCjsResolutionMode::Disabled { + if let Some(pkg_json) = + self.pkg_json_resolver.get_closest_package_json(specifier)? + { + let is_cjs_type = pkg_json.typ == "commonjs" + || self.mode == IsCjsResolutionMode::ImplicitTypeCommonJs + && pkg_json.typ == "none"; + Ok(if is_cjs_type { + ResolutionMode::Require + } else { + ResolutionMode::Import + }) + } else if self.mode == IsCjsResolutionMode::ImplicitTypeCommonJs { + Ok(ResolutionMode::Require) + } else { + Ok(ResolutionMode::Import) + } + } else { + Ok(ResolutionMode::Import) + } + } +} diff --git a/resolvers/deno/clippy.toml b/resolvers/deno/clippy.toml new file mode 100644 index 00000000000000..733ac83da1821e --- /dev/null +++ b/resolvers/deno/clippy.toml @@ -0,0 +1,52 @@ +disallowed-methods = [ + { path = "std::env::current_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::canonicalize", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::is_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::is_file", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::is_symlink", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::metadata", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::read_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::read_link", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::symlink_metadata", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::try_exists", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::exists", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::canonicalize", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::is_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::is_file", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::is_symlink", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::metadata", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::read_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::read_link", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::symlink_metadata", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::try_exists", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::env::set_current_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::env::temp_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::canonicalize", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::copy", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::create_dir_all", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::create_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::DirBuilder::new", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::hard_link", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::metadata", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::OpenOptions::new", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::read_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::read_link", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::read_to_string", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::read", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::remove_dir_all", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::remove_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::remove_file", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::rename", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::set_permissions", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::symlink_metadata", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::write", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::canonicalize", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::exists", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "url::Url::to_file_path", reason = "Use deno_path_util instead so it works in Wasm" }, + { path = "url::Url::from_file_path", reason = "Use deno_path_util instead so it works in Wasm" }, + { path = "url::Url::from_directory_path", reason = "Use deno_path_util instead so it works in Wasm" }, +] +disallowed-types = [ + # todo(dsherret): consider for the future + # { path = "std::sync::Arc", reason = "use crate::sync::MaybeArc instead" }, +] diff --git a/resolvers/deno/fs.rs b/resolvers/deno/fs.rs new file mode 100644 index 00000000000000..f2021a73a9ca87 --- /dev/null +++ b/resolvers/deno/fs.rs @@ -0,0 +1,22 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::borrow::Cow; +use std::path::Path; +use std::path::PathBuf; + +pub struct DirEntry { + pub name: String, + pub is_file: bool, + pub is_directory: bool, +} + +pub trait DenoResolverFs { + fn read_to_string_lossy( + &self, + path: &Path, + ) -> std::io::Result>; + fn realpath_sync(&self, path: &Path) -> std::io::Result; + fn exists_sync(&self, path: &Path) -> bool; + fn is_dir_sync(&self, path: &Path) -> bool; + fn read_dir_sync(&self, dir_path: &Path) -> std::io::Result>; +} diff --git a/resolvers/deno/lib.rs b/resolvers/deno/lib.rs new file mode 100644 index 00000000000000..661caf836df436 --- /dev/null +++ b/resolvers/deno/lib.rs @@ -0,0 +1,418 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +#![deny(clippy::print_stderr)] +#![deny(clippy::print_stdout)] + +use std::path::PathBuf; +use std::sync::Arc; + +use boxed_error::Boxed; +use deno_config::workspace::MappedResolution; +use deno_config::workspace::MappedResolutionDiagnostic; +use deno_config::workspace::MappedResolutionError; +use deno_config::workspace::WorkspaceResolvePkgJsonFolderError; +use deno_config::workspace::WorkspaceResolver; +use deno_package_json::PackageJsonDepValue; +use deno_package_json::PackageJsonDepValueParseError; +use deno_semver::npm::NpmPackageReqReference; +use fs::DenoResolverFs; +use node_resolver::env::NodeResolverEnv; +use node_resolver::errors::NodeResolveError; +use node_resolver::errors::PackageSubpathResolveError; +use node_resolver::InNpmPackageChecker; +use node_resolver::NodeResolution; +use node_resolver::NodeResolutionKind; +use node_resolver::NodeResolver; +use node_resolver::ResolutionMode; +use npm::MissingPackageNodeModulesFolderError; +use npm::NodeModulesOutOfDateError; +use npm::NpmReqResolver; +use npm::ResolveIfForNpmPackageErrorKind; +use npm::ResolvePkgFolderFromDenoReqError; +use npm::ResolveReqWithSubPathErrorKind; +use sloppy_imports::SloppyImportResolverFs; +use sloppy_imports::SloppyImportsResolutionKind; +use sloppy_imports::SloppyImportsResolver; +use thiserror::Error; +use url::Url; + +pub mod cjs; +pub mod fs; +pub mod npm; +pub mod sloppy_imports; + +#[derive(Debug, Clone)] +pub struct DenoResolution { + pub url: Url, + pub maybe_diagnostic: Option>, + pub found_package_json_dep: bool, +} + +#[derive(Debug, Boxed)] +pub struct DenoResolveError(pub Box); + +#[derive(Debug, Error)] +pub enum DenoResolveErrorKind { + #[error("Importing from the vendor directory is not permitted. Use a remote specifier instead or disable vendoring.")] + InvalidVendorFolderImport, + #[error(transparent)] + MappedResolution(#[from] MappedResolutionError), + #[error(transparent)] + MissingPackageNodeModulesFolder(#[from] MissingPackageNodeModulesFolderError), + #[error(transparent)] + Node(#[from] NodeResolveError), + #[error(transparent)] + NodeModulesOutOfDate(#[from] NodeModulesOutOfDateError), + #[error(transparent)] + PackageJsonDepValueParse(#[from] PackageJsonDepValueParseError), + #[error(transparent)] + PackageJsonDepValueUrlParse(url::ParseError), + #[error(transparent)] + PackageSubpathResolve(#[from] PackageSubpathResolveError), + #[error(transparent)] + ResolvePkgFolderFromDenoReq(#[from] ResolvePkgFolderFromDenoReqError), + #[error(transparent)] + WorkspaceResolvePkgJsonFolder(#[from] WorkspaceResolvePkgJsonFolderError), +} + +#[derive(Debug)] +pub struct NodeAndNpmReqResolver< + Fs: DenoResolverFs, + TNodeResolverEnv: NodeResolverEnv, +> { + pub node_resolver: Arc>, + pub npm_req_resolver: Arc>, +} + +pub struct DenoResolverOptions< + 'a, + Fs: DenoResolverFs, + TNodeResolverEnv: NodeResolverEnv, + TSloppyImportResolverFs: SloppyImportResolverFs, +> { + pub in_npm_pkg_checker: Arc, + pub node_and_req_resolver: + Option>, + pub sloppy_imports_resolver: + Option>>, + pub workspace_resolver: Arc, + /// Whether "bring your own node_modules" is enabled where Deno does not + /// setup the node_modules directories automatically, but instead uses + /// what already exists on the file system. + pub is_byonm: bool, + pub maybe_vendor_dir: Option<&'a PathBuf>, +} + +/// A resolver that takes care of resolution, taking into account loaded +/// import map, JSX settings. +#[derive(Debug)] +pub struct DenoResolver< + Fs: DenoResolverFs, + TNodeResolverEnv: NodeResolverEnv, + TSloppyImportResolverFs: SloppyImportResolverFs, +> { + in_npm_pkg_checker: Arc, + node_and_npm_resolver: Option>, + sloppy_imports_resolver: + Option>>, + workspace_resolver: Arc, + is_byonm: bool, + maybe_vendor_specifier: Option, +} + +impl< + Fs: DenoResolverFs, + TNodeResolverEnv: NodeResolverEnv, + TSloppyImportResolverFs: SloppyImportResolverFs, + > DenoResolver +{ + pub fn new( + options: DenoResolverOptions, + ) -> Self { + Self { + in_npm_pkg_checker: options.in_npm_pkg_checker, + node_and_npm_resolver: options.node_and_req_resolver, + sloppy_imports_resolver: options.sloppy_imports_resolver, + workspace_resolver: options.workspace_resolver, + is_byonm: options.is_byonm, + maybe_vendor_specifier: options + .maybe_vendor_dir + .and_then(|v| deno_path_util::url_from_directory_path(v).ok()), + } + } + + pub fn resolve( + &self, + raw_specifier: &str, + referrer: &Url, + resolution_mode: ResolutionMode, + resolution_kind: NodeResolutionKind, + ) -> Result { + let mut found_package_json_dep = false; + let mut maybe_diagnostic = None; + // Use node resolution if we're in an npm package + if let Some(node_and_npm_resolver) = self.node_and_npm_resolver.as_ref() { + let node_resolver = &node_and_npm_resolver.node_resolver; + if referrer.scheme() == "file" + && self.in_npm_pkg_checker.in_npm_package(referrer) + { + return node_resolver + .resolve(raw_specifier, referrer, resolution_mode, resolution_kind) + .map(|res| DenoResolution { + url: res.into_url(), + found_package_json_dep, + maybe_diagnostic, + }) + .map_err(|e| e.into()); + } + } + + // Attempt to resolve with the workspace resolver + let result: Result<_, DenoResolveError> = self + .workspace_resolver + .resolve(raw_specifier, referrer) + .map_err(|err| err.into()); + let result = match result { + Ok(resolution) => match resolution { + MappedResolution::Normal { + specifier, + maybe_diagnostic: current_diagnostic, + } + | MappedResolution::ImportMap { + specifier, + maybe_diagnostic: current_diagnostic, + } => { + maybe_diagnostic = current_diagnostic; + // do sloppy imports resolution if enabled + if let Some(sloppy_imports_resolver) = &self.sloppy_imports_resolver { + Ok( + sloppy_imports_resolver + .resolve( + &specifier, + match resolution_kind { + NodeResolutionKind::Execution => { + SloppyImportsResolutionKind::Execution + } + NodeResolutionKind::Types => { + SloppyImportsResolutionKind::Types + } + }, + ) + .map(|s| s.into_specifier()) + .unwrap_or(specifier), + ) + } else { + Ok(specifier) + } + } + MappedResolution::WorkspaceJsrPackage { specifier, .. } => { + Ok(specifier) + } + MappedResolution::WorkspaceNpmPackage { + target_pkg_json: pkg_json, + sub_path, + .. + } => self + .node_and_npm_resolver + .as_ref() + .unwrap() + .node_resolver + .resolve_package_subpath_from_deno_module( + pkg_json.dir_path(), + sub_path.as_deref(), + Some(referrer), + resolution_mode, + resolution_kind, + ) + .map_err(|e| e.into()), + MappedResolution::PackageJson { + dep_result, + alias, + sub_path, + .. + } => { + // found a specifier in the package.json, so mark that + // we need to do an "npm install" later + found_package_json_dep = true; + + dep_result + .as_ref() + .map_err(|e| { + DenoResolveErrorKind::PackageJsonDepValueParse(e.clone()) + .into_box() + }) + .and_then(|dep| match dep { + // todo(dsherret): it seems bad that we're converting this + // to a url because the req might not be a valid url. + PackageJsonDepValue::Req(req) => Url::parse(&format!( + "npm:{}{}", + req, + sub_path.map(|s| format!("/{}", s)).unwrap_or_default() + )) + .map_err(|e| { + DenoResolveErrorKind::PackageJsonDepValueUrlParse(e).into_box() + }), + PackageJsonDepValue::Workspace(version_req) => self + .workspace_resolver + .resolve_workspace_pkg_json_folder_for_pkg_json_dep( + alias, + version_req, + ) + .map_err(|e| { + DenoResolveErrorKind::WorkspaceResolvePkgJsonFolder(e) + .into_box() + }) + .and_then(|pkg_folder| { + self + .node_and_npm_resolver + .as_ref() + .unwrap() + .node_resolver + .resolve_package_subpath_from_deno_module( + pkg_folder, + sub_path.as_deref(), + Some(referrer), + resolution_mode, + resolution_kind, + ) + .map_err(|e| { + DenoResolveErrorKind::PackageSubpathResolve(e).into_box() + }) + }), + }) + } + }, + Err(err) => Err(err), + }; + + // When the user is vendoring, don't allow them to import directly from the vendor/ directory + // as it might cause them confusion or duplicate dependencies. Additionally, this folder has + // special treatment in the language server so it will definitely cause issues/confusion there + // if they do this. + if let Some(vendor_specifier) = &self.maybe_vendor_specifier { + if let Ok(specifier) = &result { + if specifier.as_str().starts_with(vendor_specifier.as_str()) { + return Err( + DenoResolveErrorKind::InvalidVendorFolderImport.into_box(), + ); + } + } + } + + let Some(NodeAndNpmReqResolver { + node_resolver, + npm_req_resolver, + }) = &self.node_and_npm_resolver + else { + return Ok(DenoResolution { + url: result?, + maybe_diagnostic, + found_package_json_dep, + }); + }; + + match result { + Ok(specifier) => { + if let Ok(npm_req_ref) = + NpmPackageReqReference::from_specifier(&specifier) + { + // check if the npm specifier resolves to a workspace member + if let Some(pkg_folder) = self + .workspace_resolver + .resolve_workspace_pkg_json_folder_for_npm_specifier( + npm_req_ref.req(), + ) + { + return node_resolver + .resolve_package_subpath_from_deno_module( + pkg_folder, + npm_req_ref.sub_path(), + Some(referrer), + resolution_mode, + resolution_kind, + ) + .map(|url| DenoResolution { + url, + maybe_diagnostic, + found_package_json_dep, + }) + .map_err(|e| e.into()); + } + + // do npm resolution for byonm + if self.is_byonm { + return npm_req_resolver + .resolve_req_reference( + &npm_req_ref, + referrer, + resolution_mode, + resolution_kind, + ) + .map(|url| DenoResolution { + url, + maybe_diagnostic, + found_package_json_dep, + }) + .map_err(|err| { + match err.into_kind() { + ResolveReqWithSubPathErrorKind::MissingPackageNodeModulesFolder( + err, + ) => err.into(), + ResolveReqWithSubPathErrorKind::ResolvePkgFolderFromDenoReq( + err, + ) => err.into(), + ResolveReqWithSubPathErrorKind::PackageSubpathResolve(err) => { + err.into() + } + } + }); + } + } + + Ok(DenoResolution { + url: node_resolver + .handle_if_in_node_modules(&specifier) + .unwrap_or(specifier), + maybe_diagnostic, + found_package_json_dep, + }) + } + Err(err) => { + // If byonm, check if the bare specifier resolves to an npm package + if self.is_byonm && referrer.scheme() == "file" { + let maybe_resolution = npm_req_resolver + .resolve_if_for_npm_pkg( + raw_specifier, + referrer, + resolution_mode, + resolution_kind, + ) + .map_err(|e| match e.into_kind() { + ResolveIfForNpmPackageErrorKind::NodeResolve(e) => { + DenoResolveErrorKind::Node(e).into_box() + } + ResolveIfForNpmPackageErrorKind::NodeModulesOutOfDate(e) => { + e.into() + } + })?; + if let Some(res) = maybe_resolution { + match res { + NodeResolution::Module(url) => { + return Ok(DenoResolution { + url, + maybe_diagnostic, + found_package_json_dep, + }) + } + NodeResolution::BuiltIn(_) => { + // don't resolve bare specifiers for built-in modules via node resolution + } + } + } + } + + Err(err) + } + } + } +} diff --git a/resolvers/deno/npm/byonm.rs b/resolvers/deno/npm/byonm.rs new file mode 100644 index 00000000000000..e9182d47a1fdb4 --- /dev/null +++ b/resolvers/deno/npm/byonm.rs @@ -0,0 +1,398 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::borrow::Cow; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use deno_package_json::PackageJson; +use deno_package_json::PackageJsonDepValue; +use deno_path_util::url_to_file_path; +use deno_semver::package::PackageReq; +use deno_semver::Version; +use node_resolver::env::NodeResolverEnv; +use node_resolver::errors::PackageFolderResolveError; +use node_resolver::errors::PackageFolderResolveIoError; +use node_resolver::errors::PackageJsonLoadError; +use node_resolver::errors::PackageNotFoundError; +use node_resolver::InNpmPackageChecker; +use node_resolver::NpmPackageFolderResolver; +use node_resolver::PackageJsonResolverRc; +use thiserror::Error; +use url::Url; + +use crate::fs::DenoResolverFs; + +use super::local::normalize_pkg_name_for_node_modules_deno_folder; +use super::CliNpmReqResolver; +use super::ResolvePkgFolderFromDenoReqError; + +#[derive(Debug, Error)] +pub enum ByonmResolvePkgFolderFromDenoReqError { + #[error("Could not find \"{}\" in a node_modules folder. Deno expects the node_modules/ directory to be up to date. Did you forget to run `deno install`?", .0)] + MissingAlias(String), + #[error(transparent)] + PackageJson(#[from] PackageJsonLoadError), + #[error("Could not find a matching package for 'npm:{}' in the node_modules directory. Ensure you have all your JSR and npm dependencies listed in your deno.json or package.json, then run `deno install`. Alternatively, turn on auto-install by specifying `\"nodeModulesDir\": \"auto\"` in your deno.json file.", .0)] + UnmatchedReq(PackageReq), + #[error(transparent)] + Io(#[from] std::io::Error), +} + +pub struct ByonmNpmResolverCreateOptions< + Fs: DenoResolverFs, + TEnv: NodeResolverEnv, +> { + // todo(dsherret): investigate removing this + pub root_node_modules_dir: Option, + pub fs: Fs, + pub pkg_json_resolver: PackageJsonResolverRc, +} + +#[derive(Debug)] +pub struct ByonmNpmResolver { + fs: Fs, + pkg_json_resolver: PackageJsonResolverRc, + root_node_modules_dir: Option, +} + +impl Clone + for ByonmNpmResolver +{ + fn clone(&self) -> Self { + Self { + fs: self.fs.clone(), + pkg_json_resolver: self.pkg_json_resolver.clone(), + root_node_modules_dir: self.root_node_modules_dir.clone(), + } + } +} + +impl ByonmNpmResolver { + pub fn new(options: ByonmNpmResolverCreateOptions) -> Self { + Self { + root_node_modules_dir: options.root_node_modules_dir, + fs: options.fs, + pkg_json_resolver: options.pkg_json_resolver, + } + } + + pub fn root_node_modules_dir(&self) -> Option<&Path> { + self.root_node_modules_dir.as_deref() + } + + fn load_pkg_json( + &self, + path: &Path, + ) -> Result>, PackageJsonLoadError> { + self.pkg_json_resolver.load_package_json(path) + } + + /// Finds the ancestor package.json that contains the specified dependency. + pub fn find_ancestor_package_json_with_dep( + &self, + dep_name: &str, + referrer: &Url, + ) -> Option> { + let referrer_path = url_to_file_path(referrer).ok()?; + let mut current_folder = referrer_path.parent()?; + loop { + let pkg_json_path = current_folder.join("package.json"); + if let Ok(Some(pkg_json)) = self.load_pkg_json(&pkg_json_path) { + if let Some(deps) = &pkg_json.dependencies { + if deps.contains_key(dep_name) { + return Some(pkg_json); + } + } + if let Some(deps) = &pkg_json.dev_dependencies { + if deps.contains_key(dep_name) { + return Some(pkg_json); + } + } + } + + if let Some(parent) = current_folder.parent() { + current_folder = parent; + } else { + return None; + } + } + } + + pub fn resolve_pkg_folder_from_deno_module_req( + &self, + req: &PackageReq, + referrer: &Url, + ) -> Result { + fn node_resolve_dir( + fs: &Fs, + alias: &str, + start_dir: &Path, + ) -> std::io::Result> { + for ancestor in start_dir.ancestors() { + let node_modules_folder = ancestor.join("node_modules"); + let sub_dir = join_package_name(&node_modules_folder, alias); + if fs.is_dir_sync(&sub_dir) { + return Ok(Some(deno_path_util::canonicalize_path_maybe_not_exists( + &sub_dir, + &|path| fs.realpath_sync(path), + )?)); + } + } + Ok(None) + } + + // now attempt to resolve if it's found in any package.json + let maybe_pkg_json_and_alias = + self.resolve_pkg_json_and_alias_for_req(req, referrer)?; + match maybe_pkg_json_and_alias { + Some((pkg_json, alias)) => { + // now try node resolution + if let Some(resolved) = + node_resolve_dir(&self.fs, &alias, pkg_json.dir_path())? + { + return Ok(resolved); + } + + Err(ByonmResolvePkgFolderFromDenoReqError::MissingAlias(alias)) + } + None => { + // now check if node_modules/.deno/ matches this constraint + if let Some(folder) = self.resolve_folder_in_root_node_modules(req) { + return Ok(folder); + } + + Err(ByonmResolvePkgFolderFromDenoReqError::UnmatchedReq( + req.clone(), + )) + } + } + } + + fn resolve_pkg_json_and_alias_for_req( + &self, + req: &PackageReq, + referrer: &Url, + ) -> Result, String)>, PackageJsonLoadError> { + fn resolve_alias_from_pkg_json( + req: &PackageReq, + pkg_json: &PackageJson, + ) -> Option { + let deps = pkg_json.resolve_local_package_json_deps(); + for (key, value) in deps { + if let Ok(value) = value { + match value { + PackageJsonDepValue::Req(dep_req) => { + if dep_req.name == req.name + && dep_req.version_req.intersects(&req.version_req) + { + return Some(key); + } + } + PackageJsonDepValue::Workspace(_workspace) => { + if key == req.name && req.version_req.tag() == Some("workspace") { + return Some(key); + } + } + } + } + } + None + } + + // attempt to resolve the npm specifier from the referrer's package.json, + if let Ok(file_path) = url_to_file_path(referrer) { + let mut current_path = file_path.as_path(); + while let Some(dir_path) = current_path.parent() { + let package_json_path = dir_path.join("package.json"); + if let Some(pkg_json) = self.load_pkg_json(&package_json_path)? { + if let Some(alias) = + resolve_alias_from_pkg_json(req, pkg_json.as_ref()) + { + return Ok(Some((pkg_json, alias))); + } + } + current_path = dir_path; + } + } + + // otherwise, fall fallback to the project's package.json + if let Some(root_node_modules_dir) = &self.root_node_modules_dir { + let root_pkg_json_path = + root_node_modules_dir.parent().unwrap().join("package.json"); + if let Some(pkg_json) = self.load_pkg_json(&root_pkg_json_path)? { + if let Some(alias) = resolve_alias_from_pkg_json(req, pkg_json.as_ref()) + { + return Ok(Some((pkg_json, alias))); + } + } + } + + Ok(None) + } + + fn resolve_folder_in_root_node_modules( + &self, + req: &PackageReq, + ) -> Option { + // now check if node_modules/.deno/ matches this constraint + let root_node_modules_dir = self.root_node_modules_dir.as_ref()?; + let node_modules_deno_dir = root_node_modules_dir.join(".deno"); + let Ok(entries) = self.fs.read_dir_sync(&node_modules_deno_dir) else { + return None; + }; + let search_prefix = format!( + "{}@", + normalize_pkg_name_for_node_modules_deno_folder(&req.name) + ); + let mut best_version = None; + + // example entries: + // - @denotest+add@1.0.0 + // - @denotest+add@1.0.0_1 + for entry in entries { + if !entry.is_directory { + continue; + } + let Some(version_and_copy_idx) = entry.name.strip_prefix(&search_prefix) + else { + continue; + }; + let version = version_and_copy_idx + .rsplit_once('_') + .map(|(v, _)| v) + .unwrap_or(version_and_copy_idx); + let Ok(version) = Version::parse_from_npm(version) else { + continue; + }; + if let Some(tag) = req.version_req.tag() { + let initialized_file = + node_modules_deno_dir.join(&entry.name).join(".initialized"); + let Ok(contents) = self.fs.read_to_string_lossy(&initialized_file) + else { + continue; + }; + let mut tags = contents.split(',').map(str::trim); + if tags.any(|t| t == tag) { + if let Some((best_version_version, _)) = &best_version { + if version > *best_version_version { + best_version = Some((version, entry.name)); + } + } else { + best_version = Some((version, entry.name)); + } + } + } else if req.version_req.matches(&version) { + if let Some((best_version_version, _)) = &best_version { + if version > *best_version_version { + best_version = Some((version, entry.name)); + } + } else { + best_version = Some((version, entry.name)); + } + } + } + + best_version.map(|(_version, entry_name)| { + join_package_name( + &node_modules_deno_dir.join(entry_name).join("node_modules"), + &req.name, + ) + }) + } +} + +impl< + Fs: DenoResolverFs + Send + Sync + std::fmt::Debug, + TEnv: NodeResolverEnv, + > CliNpmReqResolver for ByonmNpmResolver +{ + fn resolve_pkg_folder_from_deno_module_req( + &self, + req: &PackageReq, + referrer: &Url, + ) -> Result { + ByonmNpmResolver::resolve_pkg_folder_from_deno_module_req( + self, req, referrer, + ) + .map_err(ResolvePkgFolderFromDenoReqError::Byonm) + } +} + +impl< + Fs: DenoResolverFs + Send + Sync + std::fmt::Debug, + TEnv: NodeResolverEnv, + > NpmPackageFolderResolver for ByonmNpmResolver +{ + fn resolve_package_folder_from_package( + &self, + name: &str, + referrer: &Url, + ) -> Result { + fn inner( + fs: &Fs, + name: &str, + referrer: &Url, + ) -> Result { + let maybe_referrer_file = url_to_file_path(referrer).ok(); + let maybe_start_folder = + maybe_referrer_file.as_ref().and_then(|f| f.parent()); + if let Some(start_folder) = maybe_start_folder { + for current_folder in start_folder.ancestors() { + let node_modules_folder = if current_folder.ends_with("node_modules") + { + Cow::Borrowed(current_folder) + } else { + Cow::Owned(current_folder.join("node_modules")) + }; + + let sub_dir = join_package_name(&node_modules_folder, name); + if fs.is_dir_sync(&sub_dir) { + return Ok(sub_dir); + } + } + } + + Err( + PackageNotFoundError { + package_name: name.to_string(), + referrer: referrer.clone(), + referrer_extra: None, + } + .into(), + ) + } + + let path = inner(&self.fs, name, referrer)?; + self.fs.realpath_sync(&path).map_err(|err| { + PackageFolderResolveIoError { + package_name: name.to_string(), + referrer: referrer.clone(), + source: err, + } + .into() + }) + } +} + +#[derive(Debug)] +pub struct ByonmInNpmPackageChecker; + +impl InNpmPackageChecker for ByonmInNpmPackageChecker { + fn in_npm_package(&self, specifier: &Url) -> bool { + specifier.scheme() == "file" + && specifier + .path() + .to_ascii_lowercase() + .contains("/node_modules/") + } +} + +fn join_package_name(path: &Path, package_name: &str) -> PathBuf { + let mut path = path.to_path_buf(); + // ensure backslashes are used on windows + for part in package_name.split('/') { + path = path.join(part); + } + path +} diff --git a/resolvers/deno/npm/local.rs b/resolvers/deno/npm/local.rs new file mode 100644 index 00000000000000..aef476ad945bb0 --- /dev/null +++ b/resolvers/deno/npm/local.rs @@ -0,0 +1,27 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::borrow::Cow; + +/// Normalizes a package name for use at `node_modules/.deno/@[_]` +pub fn normalize_pkg_name_for_node_modules_deno_folder(name: &str) -> Cow { + let name = if name.to_lowercase() == name { + Cow::Borrowed(name) + } else { + Cow::Owned(format!("_{}", mixed_case_package_name_encode(name))) + }; + if name.starts_with('@') { + name.replace('/', "+").into() + } else { + name + } +} + +fn mixed_case_package_name_encode(name: &str) -> String { + // use base32 encoding because it's reversible and the character set + // only includes the characters within 0-9 and A-Z so it can be lower cased + base32::encode( + base32::Alphabet::Rfc4648Lower { padding: false }, + name.as_bytes(), + ) + .to_lowercase() +} diff --git a/resolvers/deno/npm/mod.rs b/resolvers/deno/npm/mod.rs new file mode 100644 index 00000000000000..83db04480a5d37 --- /dev/null +++ b/resolvers/deno/npm/mod.rs @@ -0,0 +1,276 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::fmt::Debug; +use std::path::PathBuf; +use std::sync::Arc; + +use boxed_error::Boxed; +use deno_semver::npm::NpmPackageReqReference; +use deno_semver::package::PackageReq; +use node_resolver::env::NodeResolverEnv; +use node_resolver::errors::NodeResolveError; +use node_resolver::errors::NodeResolveErrorKind; +use node_resolver::errors::PackageFolderResolveErrorKind; +use node_resolver::errors::PackageFolderResolveIoError; +use node_resolver::errors::PackageNotFoundError; +use node_resolver::errors::PackageResolveErrorKind; +use node_resolver::errors::PackageSubpathResolveError; +use node_resolver::InNpmPackageChecker; +use node_resolver::NodeResolution; +use node_resolver::NodeResolutionKind; +use node_resolver::NodeResolver; +use node_resolver::ResolutionMode; +use thiserror::Error; +use url::Url; + +use crate::fs::DenoResolverFs; + +pub use byonm::ByonmInNpmPackageChecker; +pub use byonm::ByonmNpmResolver; +pub use byonm::ByonmNpmResolverCreateOptions; +pub use byonm::ByonmResolvePkgFolderFromDenoReqError; +pub use local::normalize_pkg_name_for_node_modules_deno_folder; + +mod byonm; +mod local; + +#[derive(Debug, Error)] +#[error("Could not resolve \"{}\", but found it in a package.json. Deno expects the node_modules/ directory to be up to date. Did you forget to run `deno install`?", specifier)] +pub struct NodeModulesOutOfDateError { + pub specifier: String, +} + +#[derive(Debug, Error)] +#[error("Could not find '{}'. Deno expects the node_modules/ directory to be up to date. Did you forget to run `deno install`?", package_json_path.display())] +pub struct MissingPackageNodeModulesFolderError { + pub package_json_path: PathBuf, +} + +#[derive(Debug, Boxed)] +pub struct ResolveIfForNpmPackageError( + pub Box, +); + +#[derive(Debug, Error)] +pub enum ResolveIfForNpmPackageErrorKind { + #[error(transparent)] + NodeResolve(#[from] NodeResolveError), + #[error(transparent)] + NodeModulesOutOfDate(#[from] NodeModulesOutOfDateError), +} + +#[derive(Debug, Boxed)] +pub struct ResolveReqWithSubPathError(pub Box); + +#[derive(Debug, Error)] +pub enum ResolveReqWithSubPathErrorKind { + #[error(transparent)] + MissingPackageNodeModulesFolder(#[from] MissingPackageNodeModulesFolderError), + #[error(transparent)] + ResolvePkgFolderFromDenoReq(#[from] ResolvePkgFolderFromDenoReqError), + #[error(transparent)] + PackageSubpathResolve(#[from] PackageSubpathResolveError), +} + +#[derive(Debug, Error)] +pub enum ResolvePkgFolderFromDenoReqError { + // todo(dsherret): don't use anyhow here + #[error(transparent)] + Managed(anyhow::Error), + #[error(transparent)] + Byonm(#[from] ByonmResolvePkgFolderFromDenoReqError), +} + +// todo(dsherret): a temporary trait until we extract +// out the CLI npm resolver into here +pub trait CliNpmReqResolver: Debug + Send + Sync { + fn resolve_pkg_folder_from_deno_module_req( + &self, + req: &PackageReq, + referrer: &Url, + ) -> Result; +} + +pub struct NpmReqResolverOptions< + Fs: DenoResolverFs, + TNodeResolverEnv: NodeResolverEnv, +> { + /// The resolver when "bring your own node_modules" is enabled where Deno + /// does not setup the node_modules directories automatically, but instead + /// uses what already exists on the file system. + pub byonm_resolver: Option>>, + pub fs: Fs, + pub in_npm_pkg_checker: Arc, + pub node_resolver: Arc>, + pub npm_req_resolver: Arc, +} + +#[derive(Debug)] +pub struct NpmReqResolver +{ + byonm_resolver: Option>>, + fs: Fs, + in_npm_pkg_checker: Arc, + node_resolver: Arc>, + npm_resolver: Arc, +} + +impl + NpmReqResolver +{ + pub fn new(options: NpmReqResolverOptions) -> Self { + Self { + byonm_resolver: options.byonm_resolver, + fs: options.fs, + in_npm_pkg_checker: options.in_npm_pkg_checker, + node_resolver: options.node_resolver, + npm_resolver: options.npm_req_resolver, + } + } + + pub fn resolve_req_reference( + &self, + req_ref: &NpmPackageReqReference, + referrer: &Url, + resolution_mode: ResolutionMode, + resolution_kind: NodeResolutionKind, + ) -> Result { + self.resolve_req_with_sub_path( + req_ref.req(), + req_ref.sub_path(), + referrer, + resolution_mode, + resolution_kind, + ) + } + + pub fn resolve_req_with_sub_path( + &self, + req: &PackageReq, + sub_path: Option<&str>, + referrer: &Url, + resolution_mode: ResolutionMode, + resolution_kind: NodeResolutionKind, + ) -> Result { + let package_folder = self + .npm_resolver + .resolve_pkg_folder_from_deno_module_req(req, referrer)?; + let resolution_result = + self.node_resolver.resolve_package_subpath_from_deno_module( + &package_folder, + sub_path, + Some(referrer), + resolution_mode, + resolution_kind, + ); + match resolution_result { + Ok(url) => Ok(url), + Err(err) => { + if self.byonm_resolver.is_some() { + let package_json_path = package_folder.join("package.json"); + if !self.fs.exists_sync(&package_json_path) { + return Err( + MissingPackageNodeModulesFolderError { package_json_path }.into(), + ); + } + } + Err(err.into()) + } + } + } + + pub fn resolve_if_for_npm_pkg( + &self, + specifier: &str, + referrer: &Url, + resolution_mode: ResolutionMode, + resolution_kind: NodeResolutionKind, + ) -> Result, ResolveIfForNpmPackageError> { + let resolution_result = self.node_resolver.resolve( + specifier, + referrer, + resolution_mode, + resolution_kind, + ); + match resolution_result { + Ok(res) => Ok(Some(res)), + Err(err) => { + let err = err.into_kind(); + match err { + NodeResolveErrorKind::RelativeJoin(_) + | NodeResolveErrorKind::PackageImportsResolve(_) + | NodeResolveErrorKind::UnsupportedEsmUrlScheme(_) + | NodeResolveErrorKind::DataUrlReferrer(_) + | NodeResolveErrorKind::TypesNotFound(_) + | NodeResolveErrorKind::FinalizeResolution(_) => Err( + ResolveIfForNpmPackageErrorKind::NodeResolve(err.into()).into_box(), + ), + NodeResolveErrorKind::PackageResolve(err) => { + let err = err.into_kind(); + match err { + PackageResolveErrorKind::ClosestPkgJson(_) + | PackageResolveErrorKind::InvalidModuleSpecifier(_) + | PackageResolveErrorKind::ExportsResolve(_) + | PackageResolveErrorKind::SubpathResolve(_) => Err( + ResolveIfForNpmPackageErrorKind::NodeResolve( + NodeResolveErrorKind::PackageResolve(err.into()).into(), + ) + .into_box(), + ), + PackageResolveErrorKind::PackageFolderResolve(err) => { + match err.as_kind() { + PackageFolderResolveErrorKind::Io( + PackageFolderResolveIoError { package_name, .. }, + ) + | PackageFolderResolveErrorKind::PackageNotFound( + PackageNotFoundError { package_name, .. }, + ) => { + if self.in_npm_pkg_checker.in_npm_package(referrer) { + return Err( + ResolveIfForNpmPackageErrorKind::NodeResolve( + NodeResolveErrorKind::PackageResolve(err.into()) + .into(), + ) + .into_box(), + ); + } + if let Some(byonm_npm_resolver) = &self.byonm_resolver { + if byonm_npm_resolver + .find_ancestor_package_json_with_dep( + package_name, + referrer, + ) + .is_some() + { + return Err( + ResolveIfForNpmPackageErrorKind::NodeModulesOutOfDate( + NodeModulesOutOfDateError { + specifier: specifier.to_string(), + }, + ).into_box(), + ); + } + } + Ok(None) + } + PackageFolderResolveErrorKind::ReferrerNotFound(_) => { + if self.in_npm_pkg_checker.in_npm_package(referrer) { + return Err( + ResolveIfForNpmPackageErrorKind::NodeResolve( + NodeResolveErrorKind::PackageResolve(err.into()) + .into(), + ) + .into_box(), + ); + } + Ok(None) + } + } + } + } + } + } + } + } + } +} diff --git a/resolvers/deno/sloppy_imports.rs b/resolvers/deno/sloppy_imports.rs new file mode 100644 index 00000000000000..ccaa5474355eb2 --- /dev/null +++ b/resolvers/deno/sloppy_imports.rs @@ -0,0 +1,519 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::borrow::Cow; +use std::path::Path; +use std::path::PathBuf; + +use deno_media_type::MediaType; +use deno_path_util::url_from_file_path; +use deno_path_util::url_to_file_path; +use url::Url; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SloppyImportsFsEntry { + File, + Dir, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SloppyImportsResolution { + /// Ex. `./file.js` to `./file.ts` + JsToTs(Url), + /// Ex. `./file` to `./file.ts` + NoExtension(Url), + /// Ex. `./dir` to `./dir/index.ts` + Directory(Url), +} + +impl SloppyImportsResolution { + pub fn as_specifier(&self) -> &Url { + match self { + Self::JsToTs(specifier) => specifier, + Self::NoExtension(specifier) => specifier, + Self::Directory(specifier) => specifier, + } + } + + pub fn into_specifier(self) -> Url { + match self { + Self::JsToTs(specifier) => specifier, + Self::NoExtension(specifier) => specifier, + Self::Directory(specifier) => specifier, + } + } + + pub fn as_suggestion_message(&self) -> String { + format!("Maybe {}", self.as_base_message()) + } + + pub fn as_quick_fix_message(&self) -> String { + let message = self.as_base_message(); + let mut chars = message.chars(); + format!( + "{}{}.", + chars.next().unwrap().to_uppercase(), + chars.as_str() + ) + } + + fn as_base_message(&self) -> String { + match self { + SloppyImportsResolution::JsToTs(specifier) => { + let media_type = MediaType::from_specifier(specifier); + format!("change the extension to '{}'", media_type.as_ts_extension()) + } + SloppyImportsResolution::NoExtension(specifier) => { + let media_type = MediaType::from_specifier(specifier); + format!("add a '{}' extension", media_type.as_ts_extension()) + } + SloppyImportsResolution::Directory(specifier) => { + let file_name = specifier + .path() + .rsplit_once('/') + .map(|(_, file_name)| file_name) + .unwrap_or(specifier.path()); + format!("specify path to '{}' file in directory instead", file_name) + } + } + } +} + +/// The kind of resolution currently being done. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SloppyImportsResolutionKind { + /// Resolving for code that will be executed. + Execution, + /// Resolving for code that will be used for type information. + Types, +} + +impl SloppyImportsResolutionKind { + pub fn is_types(&self) -> bool { + *self == SloppyImportsResolutionKind::Types + } +} + +pub trait SloppyImportResolverFs { + fn stat_sync(&self, path: &Path) -> Option; + + fn is_file(&self, path: &Path) -> bool { + self.stat_sync(path) == Some(SloppyImportsFsEntry::File) + } +} + +#[derive(Debug)] +pub struct SloppyImportsResolver { + fs: Fs, +} + +impl SloppyImportsResolver { + pub fn new(fs: Fs) -> Self { + Self { fs } + } + + pub fn resolve( + &self, + specifier: &Url, + resolution_kind: SloppyImportsResolutionKind, + ) -> Option { + fn path_without_ext( + path: &Path, + media_type: MediaType, + ) -> Option> { + let old_path_str = path.to_string_lossy(); + match media_type { + MediaType::Unknown => Some(old_path_str), + _ => old_path_str + .strip_suffix(media_type.as_ts_extension()) + .map(|s| Cow::Owned(s.to_string())), + } + } + + fn media_types_to_paths( + path_no_ext: &str, + original_media_type: MediaType, + probe_media_type_types: Vec, + reason: SloppyImportsResolutionReason, + ) -> Vec<(PathBuf, SloppyImportsResolutionReason)> { + probe_media_type_types + .into_iter() + .filter(|media_type| *media_type != original_media_type) + .map(|media_type| { + ( + PathBuf::from(format!( + "{}{}", + path_no_ext, + media_type.as_ts_extension() + )), + reason, + ) + }) + .collect::>() + } + + if specifier.scheme() != "file" { + return None; + } + + let path = url_to_file_path(specifier).ok()?; + + #[derive(Clone, Copy)] + enum SloppyImportsResolutionReason { + JsToTs, + NoExtension, + Directory, + } + + let probe_paths: Vec<(PathBuf, SloppyImportsResolutionReason)> = + match self.fs.stat_sync(&path) { + Some(SloppyImportsFsEntry::File) => { + if resolution_kind.is_types() { + let media_type = MediaType::from_specifier(specifier); + // attempt to resolve the .d.ts file before the .js file + let probe_media_type_types = match media_type { + MediaType::JavaScript => { + vec![(MediaType::Dts), MediaType::JavaScript] + } + MediaType::Mjs => { + vec![MediaType::Dmts, MediaType::Dts, MediaType::Mjs] + } + MediaType::Cjs => { + vec![MediaType::Dcts, MediaType::Dts, MediaType::Cjs] + } + _ => return None, + }; + let path_no_ext = path_without_ext(&path, media_type)?; + media_types_to_paths( + &path_no_ext, + media_type, + probe_media_type_types, + SloppyImportsResolutionReason::JsToTs, + ) + } else { + return None; + } + } + entry @ None | entry @ Some(SloppyImportsFsEntry::Dir) => { + let media_type = MediaType::from_specifier(specifier); + let probe_media_type_types = match media_type { + MediaType::JavaScript => ( + if resolution_kind.is_types() { + vec![MediaType::TypeScript, MediaType::Tsx, MediaType::Dts] + } else { + vec![MediaType::TypeScript, MediaType::Tsx] + }, + SloppyImportsResolutionReason::JsToTs, + ), + MediaType::Jsx => { + (vec![MediaType::Tsx], SloppyImportsResolutionReason::JsToTs) + } + MediaType::Mjs => ( + if resolution_kind.is_types() { + vec![MediaType::Mts, MediaType::Dmts, MediaType::Dts] + } else { + vec![MediaType::Mts] + }, + SloppyImportsResolutionReason::JsToTs, + ), + MediaType::Cjs => ( + if resolution_kind.is_types() { + vec![MediaType::Cts, MediaType::Dcts, MediaType::Dts] + } else { + vec![MediaType::Cts] + }, + SloppyImportsResolutionReason::JsToTs, + ), + MediaType::TypeScript + | MediaType::Mts + | MediaType::Cts + | MediaType::Dts + | MediaType::Dmts + | MediaType::Dcts + | MediaType::Tsx + | MediaType::Json + | MediaType::Wasm + | MediaType::Css + | MediaType::SourceMap => { + return None; + } + MediaType::Unknown => ( + if resolution_kind.is_types() { + vec![ + MediaType::TypeScript, + MediaType::Tsx, + MediaType::Mts, + MediaType::Dts, + MediaType::Dmts, + MediaType::Dcts, + MediaType::JavaScript, + MediaType::Jsx, + MediaType::Mjs, + ] + } else { + vec![ + MediaType::TypeScript, + MediaType::JavaScript, + MediaType::Tsx, + MediaType::Jsx, + MediaType::Mts, + MediaType::Mjs, + ] + }, + SloppyImportsResolutionReason::NoExtension, + ), + }; + let mut probe_paths = match path_without_ext(&path, media_type) { + Some(path_no_ext) => media_types_to_paths( + &path_no_ext, + media_type, + probe_media_type_types.0, + probe_media_type_types.1, + ), + None => vec![], + }; + + if matches!(entry, Some(SloppyImportsFsEntry::Dir)) { + // try to resolve at the index file + if resolution_kind.is_types() { + probe_paths.push(( + path.join("index.ts"), + SloppyImportsResolutionReason::Directory, + )); + + probe_paths.push(( + path.join("index.mts"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.d.ts"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.d.mts"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.js"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.mjs"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.tsx"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.jsx"), + SloppyImportsResolutionReason::Directory, + )); + } else { + probe_paths.push(( + path.join("index.ts"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.mts"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.tsx"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.js"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.mjs"), + SloppyImportsResolutionReason::Directory, + )); + probe_paths.push(( + path.join("index.jsx"), + SloppyImportsResolutionReason::Directory, + )); + } + } + if probe_paths.is_empty() { + return None; + } + probe_paths + } + }; + + for (probe_path, reason) in probe_paths { + if self.fs.is_file(&probe_path) { + if let Ok(specifier) = url_from_file_path(&probe_path) { + match reason { + SloppyImportsResolutionReason::JsToTs => { + return Some(SloppyImportsResolution::JsToTs(specifier)); + } + SloppyImportsResolutionReason::NoExtension => { + return Some(SloppyImportsResolution::NoExtension(specifier)); + } + SloppyImportsResolutionReason::Directory => { + return Some(SloppyImportsResolution::Directory(specifier)); + } + } + } + } + } + + None + } +} + +#[cfg(test)] +mod test { + use test_util::TestContext; + + use super::*; + + #[test] + fn test_unstable_sloppy_imports() { + fn resolve(specifier: &Url) -> Option { + resolve_with_resolution_kind( + specifier, + SloppyImportsResolutionKind::Execution, + ) + } + + fn resolve_types(specifier: &Url) -> Option { + resolve_with_resolution_kind( + specifier, + SloppyImportsResolutionKind::Types, + ) + } + + fn resolve_with_resolution_kind( + specifier: &Url, + resolution_kind: SloppyImportsResolutionKind, + ) -> Option { + struct RealSloppyImportsResolverFs; + impl SloppyImportResolverFs for RealSloppyImportsResolverFs { + fn stat_sync(&self, path: &Path) -> Option { + #[allow(clippy::disallowed_methods)] + let stat = std::fs::metadata(path).ok()?; + if stat.is_dir() { + Some(SloppyImportsFsEntry::Dir) + } else if stat.is_file() { + Some(SloppyImportsFsEntry::File) + } else { + None + } + } + } + + SloppyImportsResolver::new(RealSloppyImportsResolverFs) + .resolve(specifier, resolution_kind) + } + + let context = TestContext::default(); + let temp_dir = context.temp_dir().path(); + + // scenarios like resolving ./example.js to ./example.ts + for (ext_from, ext_to) in [("js", "ts"), ("js", "tsx"), ("mjs", "mts")] { + let ts_file = temp_dir.join(format!("file.{}", ext_to)); + ts_file.write(""); + assert_eq!(resolve(&ts_file.url_file()), None); + assert_eq!( + resolve( + &temp_dir + .url_dir() + .join(&format!("file.{}", ext_from)) + .unwrap() + ), + Some(SloppyImportsResolution::JsToTs(ts_file.url_file())), + ); + ts_file.remove_file(); + } + + // no extension scenarios + for ext in ["js", "ts", "js", "tsx", "jsx", "mjs", "mts"] { + let file = temp_dir.join(format!("file.{}", ext)); + file.write(""); + assert_eq!( + resolve( + &temp_dir + .url_dir() + .join("file") // no ext + .unwrap() + ), + Some(SloppyImportsResolution::NoExtension(file.url_file())) + ); + file.remove_file(); + } + + // .ts and .js exists, .js specified (goes to specified) + { + let ts_file = temp_dir.join("file.ts"); + ts_file.write(""); + let js_file = temp_dir.join("file.js"); + js_file.write(""); + assert_eq!(resolve(&js_file.url_file()), None); + } + + // only js exists, .js specified + { + let js_only_file = temp_dir.join("js_only.js"); + js_only_file.write(""); + assert_eq!(resolve(&js_only_file.url_file()), None); + assert_eq!(resolve_types(&js_only_file.url_file()), None); + } + + // resolving a directory to an index file + { + let routes_dir = temp_dir.join("routes"); + routes_dir.create_dir_all(); + let index_file = routes_dir.join("index.ts"); + index_file.write(""); + assert_eq!( + resolve(&routes_dir.url_file()), + Some(SloppyImportsResolution::Directory(index_file.url_file())), + ); + } + + // both a directory and a file with specifier is present + { + let api_dir = temp_dir.join("api"); + api_dir.create_dir_all(); + let bar_file = api_dir.join("bar.ts"); + bar_file.write(""); + let api_file = temp_dir.join("api.ts"); + api_file.write(""); + assert_eq!( + resolve(&api_dir.url_file()), + Some(SloppyImportsResolution::NoExtension(api_file.url_file())), + ); + } + } + + #[test] + fn test_sloppy_import_resolution_suggestion_message() { + // directory + assert_eq!( + SloppyImportsResolution::Directory( + Url::parse("file:///dir/index.js").unwrap() + ) + .as_suggestion_message(), + "Maybe specify path to 'index.js' file in directory instead" + ); + // no ext + assert_eq!( + SloppyImportsResolution::NoExtension( + Url::parse("file:///dir/index.mjs").unwrap() + ) + .as_suggestion_message(), + "Maybe add a '.mjs' extension" + ); + // js to ts + assert_eq!( + SloppyImportsResolution::JsToTs( + Url::parse("file:///dir/index.mts").unwrap() + ) + .as_suggestion_message(), + "Maybe change the extension to '.mts'" + ); + } +} diff --git a/resolvers/node/Cargo.toml b/resolvers/node/Cargo.toml new file mode 100644 index 00000000000000..40fd5b87aa0789 --- /dev/null +++ b/resolvers/node/Cargo.toml @@ -0,0 +1,34 @@ +# Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +[package] +name = "node_resolver" +version = "0.19.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +readme = "README.md" +repository.workspace = true +description = "Node.js module resolution algorithm used in Deno" + +[lib] +path = "lib.rs" + +[features] +sync = ["deno_package_json/sync"] + +[dependencies] +anyhow.workspace = true +async-trait.workspace = true +boxed_error.workspace = true +deno_media_type.workspace = true +deno_package_json.workspace = true +deno_path_util.workspace = true +futures.workspace = true +lazy-regex.workspace = true +once_cell.workspace = true +path-clean = "=0.1.0" +regex.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tokio.workspace = true +url.workspace = true diff --git a/ext/node_resolver/README.md b/resolvers/node/README.md similarity index 100% rename from ext/node_resolver/README.md rename to resolvers/node/README.md diff --git a/ext/node_resolver/analyze.rs b/resolvers/node/analyze.rs similarity index 85% rename from ext/node_resolver/analyze.rs rename to resolvers/node/analyze.rs index deb56d064ef909..a444f4d923d0eb 100644 --- a/ext/node_resolver/analyze.rs +++ b/resolvers/node/analyze.rs @@ -6,6 +6,8 @@ use std::collections::HashSet; use std::path::Path; use std::path::PathBuf; +use deno_path_util::url_from_file_path; +use deno_path_util::url_to_file_path; use futures::future::LocalBoxFuture; use futures::stream::FuturesUnordered; use futures::FutureExt; @@ -17,19 +19,19 @@ use anyhow::Error as AnyError; use url::Url; use crate::env::NodeResolverEnv; -use crate::package_json::load_pkg_json; -use crate::path::to_file_specifier; +use crate::npm::InNpmPackageCheckerRc; use crate::resolution::NodeResolverRc; -use crate::NodeModuleKind; -use crate::NodeResolutionMode; -use crate::NpmResolverRc; +use crate::NodeResolutionKind; +use crate::NpmPackageFolderResolverRc; +use crate::PackageJsonResolverRc; use crate::PathClean; +use crate::ResolutionMode; #[derive(Debug, Clone)] -pub enum CjsAnalysis { +pub enum CjsAnalysis<'a> { /// File was found to be an ES module and the translator should /// load the code as ESM. - Esm(String), + Esm(Cow<'a, str>), Cjs(CjsAnalysisExports), } @@ -49,11 +51,11 @@ pub trait CjsCodeAnalyzer { /// already has it. If the source is needed by the implementation, /// then it can use the provided source, or otherwise load it if /// necessary. - async fn analyze_cjs( + async fn analyze_cjs<'a>( &self, specifier: &Url, - maybe_source: Option, - ) -> Result; + maybe_source: Option>, + ) -> Result, AnyError>; } pub struct NodeCodeTranslator< @@ -62,8 +64,10 @@ pub struct NodeCodeTranslator< > { cjs_code_analyzer: TCjsCodeAnalyzer, env: TNodeResolverEnv, + in_npm_pkg_checker: InNpmPackageCheckerRc, node_resolver: NodeResolverRc, - npm_resolver: NpmResolverRc, + npm_resolver: NpmPackageFolderResolverRc, + pkg_json_resolver: PackageJsonResolverRc, } impl @@ -72,14 +76,18 @@ impl pub fn new( cjs_code_analyzer: TCjsCodeAnalyzer, env: TNodeResolverEnv, + in_npm_pkg_checker: InNpmPackageCheckerRc, node_resolver: NodeResolverRc, - npm_resolver: NpmResolverRc, + npm_resolver: NpmPackageFolderResolverRc, + pkg_json_resolver: PackageJsonResolverRc, ) -> Self { Self { cjs_code_analyzer, env, + in_npm_pkg_checker, node_resolver, npm_resolver, + pkg_json_resolver, } } @@ -89,11 +97,11 @@ impl /// For all discovered reexports the analysis will be performed recursively. /// /// If successful a source code for equivalent ES module is returned. - pub async fn translate_cjs_to_esm( + pub async fn translate_cjs_to_esm<'a>( &self, entry_specifier: &Url, - source: Option, - ) -> Result { + source: Option>, + ) -> Result, AnyError> { let mut temp_var_count = 0; let analysis = self @@ -107,7 +115,7 @@ impl }; let mut source = vec![ - r#"import {createRequire as __internalCreateRequire} from "node:module"; + r#"import {createRequire as __internalCreateRequire, Module as __internalModule } from "node:module"; const require = __internalCreateRequire(import.meta.url);"# .to_string(), ]; @@ -134,9 +142,13 @@ impl } source.push(format!( - "const mod = require(\"{}\");", - entry_specifier - .to_file_path() + r#"let mod; + if (import.meta.main) {{ + mod = __internalModule._load("{0}", null, true) + }} else {{ + mod = require("{0}"); + }}"#, + url_to_file_path(entry_specifier) .unwrap() .to_str() .unwrap() @@ -159,7 +171,7 @@ impl source.push("export default mod;".to_string()); let translated_source = source.join("\n"); - Ok(translated_source) + Ok(Cow::Owned(translated_source)) } async fn analyze_reexports<'a>( @@ -174,7 +186,7 @@ impl struct Analysis { reexport_specifier: url::Url, referrer: url::Url, - analysis: CjsAnalysis, + analysis: CjsAnalysis<'static>, } type AnalysisFuture<'a> = LocalBoxFuture<'a, Result>; @@ -197,7 +209,7 @@ impl // FIXME(bartlomieju): check if these conditions are okay, probably // should be `deno-require`, because `deno` is already used in `esm_resolver.rs` &["deno", "node", "require", "default"], - NodeResolutionMode::Execution, + NodeResolutionKind::Execution, ); let reexport_specifier = match result { Ok(Some(specifier)) => specifier, @@ -291,21 +303,19 @@ impl specifier: &str, referrer: &Url, conditions: &[&str], - mode: NodeResolutionMode, + resolution_kind: NodeResolutionKind, ) -> Result, AnyError> { if specifier.starts_with('/') { todo!(); } - let referrer_path = referrer.to_file_path().unwrap(); + let referrer_path = url_to_file_path(referrer).unwrap(); if specifier.starts_with("./") || specifier.starts_with("../") { if let Some(parent) = referrer_path.parent() { - return Some( - self - .file_extension_probe(parent.join(specifier), &referrer_path) - .map(|p| to_file_specifier(&p)), - ) - .transpose(); + return self + .file_extension_probe(parent.join(specifier), &referrer_path) + .and_then(|p| url_from_file_path(&p).map_err(AnyError::from)) + .map(Some); } else { todo!(); } @@ -331,8 +341,9 @@ impl }?; let package_json_path = module_dir.join("package.json"); - let maybe_package_json = - load_pkg_json(self.env.pkg_json_fs(), &package_json_path)?; + let maybe_package_json = self + .pkg_json_resolver + .load_package_json(&package_json_path)?; if let Some(package_json) = maybe_package_json { if let Some(exports) = &package_json.exports { return Some( @@ -343,9 +354,9 @@ impl &package_subpath, exports, Some(referrer), - NodeModuleKind::Esm, + ResolutionMode::Import, conditions, - mode, + resolution_kind, ) .map_err(AnyError::from), ) @@ -358,35 +369,38 @@ impl if self.env.is_dir_sync(&d) { // subdir might have a package.json that specifies the entrypoint let package_json_path = d.join("package.json"); - let maybe_package_json = - load_pkg_json(self.env.pkg_json_fs(), &package_json_path)?; + let maybe_package_json = self + .pkg_json_resolver + .load_package_json(&package_json_path)?; if let Some(package_json) = maybe_package_json { - if let Some(main) = package_json.main(NodeModuleKind::Cjs) { - return Ok(Some(to_file_specifier(&d.join(main).clean()))); + if let Some(main) = + package_json.main(deno_package_json::NodeModuleKind::Cjs) + { + return Ok(Some(url_from_file_path(&d.join(main).clean())?)); } } - return Ok(Some(to_file_specifier(&d.join("index.js").clean()))); + return Ok(Some(url_from_file_path(&d.join("index.js").clean())?)); } - return Some( - self - .file_extension_probe(d, &referrer_path) - .map(|p| to_file_specifier(&p)), - ) - .transpose(); - } else if let Some(main) = package_json.main(NodeModuleKind::Cjs) { - return Ok(Some(to_file_specifier(&module_dir.join(main).clean()))); + return self + .file_extension_probe(d, &referrer_path) + .and_then(|p| url_from_file_path(&p).map_err(AnyError::from)) + .map(Some); + } else if let Some(main) = + package_json.main(deno_package_json::NodeModuleKind::Cjs) + { + return Ok(Some(url_from_file_path(&module_dir.join(main).clean())?)); } else { - return Ok(Some(to_file_specifier( + return Ok(Some(url_from_file_path( &module_dir.join("index.js").clean(), - ))); + )?)); } } // as a fallback, attempt to resolve it via the ancestor directories let mut last = referrer_path.as_path(); while let Some(parent) = last.parent() { - if !self.npm_resolver.in_npm_package_at_dir_path(parent) { + if !self.in_npm_pkg_checker.in_npm_package_at_dir_path(parent) { break; } let path = if parent.ends_with("node_modules") { @@ -395,7 +409,7 @@ impl parent.join("node_modules").join(specifier) }; if let Ok(path) = self.file_extension_probe(path, &referrer_path) { - return Ok(Some(to_file_specifier(&path))); + return Ok(Some(url_from_file_path(&path)?)); } last = parent; } diff --git a/resolvers/node/clippy.toml b/resolvers/node/clippy.toml new file mode 100644 index 00000000000000..90eaba3fae2a8f --- /dev/null +++ b/resolvers/node/clippy.toml @@ -0,0 +1,51 @@ +disallowed-methods = [ + { path = "std::env::current_dir", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::Path::canonicalize", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::Path::is_dir", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::Path::is_file", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::Path::is_symlink", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::Path::metadata", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::Path::read_dir", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::Path::read_link", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::Path::symlink_metadata", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::Path::try_exists", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::PathBuf::exists", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::PathBuf::canonicalize", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::PathBuf::is_dir", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::PathBuf::is_file", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::PathBuf::is_symlink", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::PathBuf::metadata", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::PathBuf::read_dir", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::PathBuf::read_link", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::PathBuf::symlink_metadata", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::PathBuf::try_exists", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::env::set_current_dir", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::env::temp_dir", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::canonicalize", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::copy", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::create_dir_all", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::create_dir", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::DirBuilder::new", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::hard_link", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::metadata", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::OpenOptions::new", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::read_dir", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::read_link", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::read_to_string", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::read", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::remove_dir_all", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::remove_dir", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::remove_file", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::rename", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::set_permissions", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::symlink_metadata", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::fs::write", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::Path::canonicalize", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "std::path::Path::exists", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "url::Url::to_file_path", reason = "Use deno_path_util instead so it works in Wasm" }, + { path = "url::Url::from_file_path", reason = "Use deno_path_util instead so it works in Wasm" }, + { path = "url::Url::from_directory_path", reason = "Use deno_path_util instead so it works in Wasm" }, +] +disallowed-types = [ + { path = "std::sync::Arc", reason = "use crate::sync::MaybeArc instead" }, +] diff --git a/ext/node_resolver/env.rs b/resolvers/node/env.rs similarity index 100% rename from ext/node_resolver/env.rs rename to resolvers/node/env.rs diff --git a/ext/node_resolver/errors.rs b/resolvers/node/errors.rs similarity index 84% rename from ext/node_resolver/errors.rs rename to resolvers/node/errors.rs index 4ba829eda54e9a..600a365a8f5c20 100644 --- a/ext/node_resolver/errors.rs +++ b/resolvers/node/errors.rs @@ -4,38 +4,12 @@ use std::borrow::Cow; use std::fmt::Write; use std::path::PathBuf; +use boxed_error::Boxed; use thiserror::Error; use url::Url; -use crate::NodeModuleKind; -use crate::NodeResolutionMode; - -macro_rules! kinded_err { - ($name:ident, $kind_name:ident) => { - #[derive(Error, Debug)] - #[error(transparent)] - pub struct $name(pub Box<$kind_name>); - - impl $name { - pub fn as_kind(&self) -> &$kind_name { - &self.0 - } - - pub fn into_kind(self) -> $kind_name { - *self.0 - } - } - - impl From for $name - where - $kind_name: From, - { - fn from(err: E) -> Self { - $name(Box::new($kind_name::from(err))) - } - } - }; -} +use crate::NodeResolutionKind; +use crate::ResolutionMode; #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] #[allow(non_camel_case_types)] @@ -81,29 +55,6 @@ pub trait NodeJsErrorCoded { fn code(&self) -> NodeJsErrorCode; } -kinded_err!( - ResolvePkgSubpathFromDenoModuleError, - ResolvePkgSubpathFromDenoModuleErrorKind -); - -impl NodeJsErrorCoded for ResolvePkgSubpathFromDenoModuleError { - fn code(&self) -> NodeJsErrorCode { - use ResolvePkgSubpathFromDenoModuleErrorKind::*; - match self.as_kind() { - PackageSubpathResolve(e) => e.code(), - UrlToNodeResolution(e) => e.code(), - } - } -} - -#[derive(Debug, Error)] -pub enum ResolvePkgSubpathFromDenoModuleErrorKind { - #[error(transparent)] - PackageSubpathResolve(#[from] PackageSubpathResolveError), - #[error(transparent)] - UrlToNodeResolution(#[from] UrlToNodeResolutionError), -} - // todo(https://github.com/denoland/deno_core/issues/810): make this a TypeError #[derive(Debug, Clone, Error)] #[error( @@ -125,7 +76,8 @@ impl NodeJsErrorCoded for InvalidModuleSpecifierError { } } -kinded_err!(LegacyResolveError, LegacyResolveErrorKind); +#[derive(Debug, Boxed)] +pub struct LegacyResolveError(pub Box); #[derive(Debug, Error)] pub enum LegacyResolveErrorKind { @@ -144,8 +96,6 @@ impl NodeJsErrorCoded for LegacyResolveError { } } -kinded_err!(PackageFolderResolveError, PackageFolderResolveErrorKind); - #[derive(Debug, Error)] #[error( "Could not find package '{}' from referrer '{}'{}.", @@ -209,6 +159,9 @@ impl NodeJsErrorCoded for PackageFolderResolveError { } } +#[derive(Debug, Boxed)] +pub struct PackageFolderResolveError(pub Box); + #[derive(Debug, Error)] pub enum PackageFolderResolveErrorKind { #[error(transparent)] @@ -219,8 +172,6 @@ pub enum PackageFolderResolveErrorKind { Io(#[from] PackageFolderResolveIoError), } -kinded_err!(PackageSubpathResolveError, PackageSubpathResolveErrorKind); - impl NodeJsErrorCoded for PackageSubpathResolveError { fn code(&self) -> NodeJsErrorCode { match self.as_kind() { @@ -231,6 +182,9 @@ impl NodeJsErrorCoded for PackageSubpathResolveError { } } +#[derive(Debug, Boxed)] +pub struct PackageSubpathResolveError(pub Box); + #[derive(Debug, Error)] pub enum PackageSubpathResolveErrorKind { #[error(transparent)] @@ -249,24 +203,24 @@ pub enum PackageSubpathResolveErrorKind { maybe_referrer.as_ref().map(|r| format!( " from{} referrer {}", - match referrer_kind { - NodeModuleKind::Esm => "", - NodeModuleKind::Cjs => " cjs", + match resolution_mode { + ResolutionMode::Import => "", + ResolutionMode::Require => " cjs", }, r ) ).unwrap_or_default(), - match mode { - NodeResolutionMode::Execution => "", - NodeResolutionMode::Types => " for types", + match resolution_kind { + NodeResolutionKind::Execution => "", + NodeResolutionKind::Types => " for types", } )] pub struct PackageTargetNotFoundError { pub pkg_json_path: PathBuf, pub target: String, pub maybe_referrer: Option, - pub referrer_kind: NodeModuleKind, - pub mode: NodeResolutionMode, + pub resolution_mode: ResolutionMode, + pub resolution_kind: NodeResolutionKind, } impl NodeJsErrorCoded for PackageTargetNotFoundError { @@ -275,8 +229,6 @@ impl NodeJsErrorCoded for PackageTargetNotFoundError { } } -kinded_err!(PackageTargetResolveError, PackageTargetResolveErrorKind); - impl NodeJsErrorCoded for PackageTargetResolveError { fn code(&self) -> NodeJsErrorCode { match self.as_kind() { @@ -289,6 +241,9 @@ impl NodeJsErrorCoded for PackageTargetResolveError { } } +#[derive(Debug, Boxed)] +pub struct PackageTargetResolveError(pub Box); + #[derive(Debug, Error)] pub enum PackageTargetResolveErrorKind { #[error(transparent)] @@ -303,8 +258,6 @@ pub enum PackageTargetResolveErrorKind { TypesNotFound(#[from] TypesNotFoundError), } -kinded_err!(PackageExportsResolveError, PackageExportsResolveErrorKind); - impl NodeJsErrorCoded for PackageExportsResolveError { fn code(&self) -> NodeJsErrorCode { match self.as_kind() { @@ -314,6 +267,9 @@ impl NodeJsErrorCoded for PackageExportsResolveError { } } +#[derive(Debug, Boxed)] +pub struct PackageExportsResolveError(pub Box); + #[derive(Debug, Error)] pub enum PackageExportsResolveErrorKind { #[error(transparent)] @@ -361,8 +317,6 @@ impl NodeJsErrorCoded for PackageJsonLoadError { } } -kinded_err!(ClosestPkgJsonError, ClosestPkgJsonErrorKind); - impl NodeJsErrorCoded for ClosestPkgJsonError { fn code(&self) -> NodeJsErrorCode { match self.as_kind() { @@ -372,6 +326,9 @@ impl NodeJsErrorCoded for ClosestPkgJsonError { } } +#[derive(Debug, Boxed)] +pub struct ClosestPkgJsonError(pub Box); + #[derive(Debug, Error)] pub enum ClosestPkgJsonErrorKind { #[error(transparent)] @@ -394,37 +351,6 @@ impl NodeJsErrorCoded for CanonicalizingPkgJsonDirError { } } -#[derive(Debug, Error)] -#[error("TypeScript files are not supported in npm packages: {specifier}")] -pub struct TypeScriptNotSupportedInNpmError { - pub specifier: Url, -} - -impl NodeJsErrorCoded for TypeScriptNotSupportedInNpmError { - fn code(&self) -> NodeJsErrorCode { - NodeJsErrorCode::ERR_UNKNOWN_FILE_EXTENSION - } -} - -kinded_err!(UrlToNodeResolutionError, UrlToNodeResolutionErrorKind); - -impl NodeJsErrorCoded for UrlToNodeResolutionError { - fn code(&self) -> NodeJsErrorCode { - match self.as_kind() { - UrlToNodeResolutionErrorKind::TypeScriptNotSupported(e) => e.code(), - UrlToNodeResolutionErrorKind::ClosestPkgJson(e) => e.code(), - } - } -} - -#[derive(Debug, Error)] -pub enum UrlToNodeResolutionErrorKind { - #[error(transparent)] - TypeScriptNotSupported(#[from] TypeScriptNotSupportedInNpmError), - #[error(transparent)] - ClosestPkgJson(#[from] ClosestPkgJsonError), -} - // todo(https://github.com/denoland/deno_core/issues/810): make this a TypeError #[derive(Debug, Error)] #[error( @@ -446,7 +372,8 @@ impl NodeJsErrorCoded for PackageImportNotDefinedError { } } -kinded_err!(PackageImportsResolveError, PackageImportsResolveErrorKind); +#[derive(Debug, Boxed)] +pub struct PackageImportsResolveError(pub Box); #[derive(Debug, Error)] pub enum PackageImportsResolveErrorKind { @@ -471,8 +398,6 @@ impl NodeJsErrorCoded for PackageImportsResolveErrorKind { } } -kinded_err!(PackageResolveError, PackageResolveErrorKind); - impl NodeJsErrorCoded for PackageResolveError { fn code(&self) -> NodeJsErrorCode { match self.as_kind() { @@ -485,6 +410,9 @@ impl NodeJsErrorCoded for PackageResolveError { } } +#[derive(Debug, Boxed)] +pub struct PackageResolveError(pub Box); + #[derive(Debug, Error)] pub enum PackageResolveErrorKind { #[error(transparent)] @@ -515,7 +443,8 @@ pub struct DataUrlReferrerError { pub source: url::ParseError, } -kinded_err!(NodeResolveError, NodeResolveErrorKind); +#[derive(Debug, Boxed)] +pub struct NodeResolveError(pub Box); #[derive(Debug, Error)] pub enum NodeResolveErrorKind { @@ -533,11 +462,10 @@ pub enum NodeResolveErrorKind { TypesNotFound(#[from] TypesNotFoundError), #[error(transparent)] FinalizeResolution(#[from] FinalizeResolutionError), - #[error(transparent)] - UrlToNodeResolution(#[from] UrlToNodeResolutionError), } -kinded_err!(FinalizeResolutionError, FinalizeResolutionErrorKind); +#[derive(Debug, Boxed)] +pub struct FinalizeResolutionError(pub Box); #[derive(Debug, Error)] pub enum FinalizeResolutionErrorKind { @@ -658,7 +586,7 @@ pub struct PackagePathNotExportedError { pub pkg_json_path: PathBuf, pub subpath: String, pub maybe_referrer: Option, - pub mode: NodeResolutionMode, + pub resolution_kind: NodeResolutionKind, } impl NodeJsErrorCoded for PackagePathNotExportedError { @@ -675,9 +603,9 @@ impl std::fmt::Display for PackagePathNotExportedError { f.write_str(self.code().as_str())?; f.write_char(']')?; - let types_msg = match self.mode { - NodeResolutionMode::Execution => String::new(), - NodeResolutionMode::Types => " for types".to_string(), + let types_msg = match self.resolution_kind { + NodeResolutionKind::Execution => String::new(), + NodeResolutionKind::Types => " for types".to_string(), }; if self.subpath == "." { write!( @@ -728,8 +656,6 @@ pub enum ResolvePkgJsonBinExportError { MissingPkgJson { pkg_json_path: PathBuf }, #[error("Failed resolving binary export. {message}")] InvalidBinProperty { message: String }, - #[error(transparent)] - UrlToNodeResolution(#[from] UrlToNodeResolutionError), } #[derive(Debug, Error)] @@ -752,7 +678,7 @@ mod test { pkg_json_path: PathBuf::from("test_path").join("package.json"), subpath: "./jsx-runtime".to_string(), maybe_referrer: None, - mode: NodeResolutionMode::Types + resolution_kind: NodeResolutionKind::Types }.to_string(), format!("[ERR_PACKAGE_PATH_NOT_EXPORTED] Package subpath './jsx-runtime' is not defined for types by \"exports\" in 'test_path{separator_char}package.json'") ); @@ -761,7 +687,7 @@ mod test { pkg_json_path: PathBuf::from("test_path").join("package.json"), subpath: ".".to_string(), maybe_referrer: None, - mode: NodeResolutionMode::Types + resolution_kind: NodeResolutionKind::Types }.to_string(), format!("[ERR_PACKAGE_PATH_NOT_EXPORTED] No \"exports\" main defined for types in 'test_path{separator_char}package.json'") ); diff --git a/resolvers/node/lib.rs b/resolvers/node/lib.rs new file mode 100644 index 00000000000000..8da20c421e8e66 --- /dev/null +++ b/resolvers/node/lib.rs @@ -0,0 +1,31 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +#![deny(clippy::print_stderr)] +#![deny(clippy::print_stdout)] + +pub mod analyze; +pub mod env; +pub mod errors; +mod npm; +mod package_json; +mod path; +mod resolution; +mod sync; + +pub use deno_package_json::PackageJson; +pub use npm::InNpmPackageChecker; +pub use npm::InNpmPackageCheckerRc; +pub use npm::NpmPackageFolderResolver; +pub use npm::NpmPackageFolderResolverRc; +pub use package_json::PackageJsonResolver; +pub use package_json::PackageJsonResolverRc; +pub use package_json::PackageJsonThreadLocalCache; +pub use path::PathClean; +pub use resolution::parse_npm_pkg_name; +pub use resolution::resolve_specifier_into_node_modules; +pub use resolution::NodeResolution; +pub use resolution::NodeResolutionKind; +pub use resolution::NodeResolver; +pub use resolution::ResolutionMode; +pub use resolution::DEFAULT_CONDITIONS; +pub use resolution::REQUIRE_CONDITIONS; diff --git a/resolvers/node/npm.rs b/resolvers/node/npm.rs new file mode 100644 index 00000000000000..ab3a1794262304 --- /dev/null +++ b/resolvers/node/npm.rs @@ -0,0 +1,52 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::path::Path; +use std::path::PathBuf; + +use deno_path_util::url_from_directory_path; +use deno_path_util::url_from_file_path; +use url::Url; + +use crate::errors; +use crate::path::PathClean; +use crate::sync::MaybeSend; +use crate::sync::MaybeSync; + +#[allow(clippy::disallowed_types)] +pub type NpmPackageFolderResolverRc = + crate::sync::MaybeArc; + +pub trait NpmPackageFolderResolver: + std::fmt::Debug + MaybeSend + MaybeSync +{ + /// Resolves an npm package folder path from the specified referrer. + fn resolve_package_folder_from_package( + &self, + specifier: &str, + referrer: &Url, + ) -> Result; +} + +#[allow(clippy::disallowed_types)] +pub type InNpmPackageCheckerRc = crate::sync::MaybeArc; + +/// Checks if a provided specifier is in an npm package. +pub trait InNpmPackageChecker: std::fmt::Debug + MaybeSend + MaybeSync { + fn in_npm_package(&self, specifier: &Url) -> bool; + + fn in_npm_package_at_dir_path(&self, path: &Path) -> bool { + let specifier = match url_from_directory_path(&path.to_path_buf().clean()) { + Ok(p) => p, + Err(_) => return false, + }; + self.in_npm_package(&specifier) + } + + fn in_npm_package_at_file_path(&self, path: &Path) -> bool { + let specifier = match url_from_file_path(&path.to_path_buf().clean()) { + Ok(p) => p, + Err(_) => return false, + }; + self.in_npm_package(&specifier) + } +} diff --git a/resolvers/node/package_json.rs b/resolvers/node/package_json.rs new file mode 100644 index 00000000000000..ae016ebe3ec3c8 --- /dev/null +++ b/resolvers/node/package_json.rs @@ -0,0 +1,129 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use deno_package_json::PackageJson; +use deno_package_json::PackageJsonRc; +use deno_path_util::strip_unc_prefix; +use std::cell::RefCell; +use std::collections::HashMap; +use std::io::ErrorKind; +use std::path::Path; +use std::path::PathBuf; +use url::Url; + +use crate::env::NodeResolverEnv; +use crate::errors::CanonicalizingPkgJsonDirError; +use crate::errors::ClosestPkgJsonError; +use crate::errors::PackageJsonLoadError; + +// it would be nice if this was passed down as a ctor arg to the package.json resolver, +// but it's a little bit complicated to do that, so we just maintain a thread local cache +thread_local! { + static CACHE: RefCell> = RefCell::new(HashMap::new()); +} + +pub struct PackageJsonThreadLocalCache; + +impl PackageJsonThreadLocalCache { + pub fn clear() { + CACHE.with(|cache| cache.borrow_mut().clear()); + } +} + +impl deno_package_json::PackageJsonCache for PackageJsonThreadLocalCache { + fn get(&self, path: &Path) -> Option { + CACHE.with(|cache| cache.borrow().get(path).cloned()) + } + + fn set(&self, path: PathBuf, package_json: PackageJsonRc) { + CACHE.with(|cache| cache.borrow_mut().insert(path, package_json)); + } +} + +#[allow(clippy::disallowed_types)] +pub type PackageJsonResolverRc = + crate::sync::MaybeArc>; + +#[derive(Debug)] +pub struct PackageJsonResolver { + env: TEnv, +} + +impl PackageJsonResolver { + pub fn new(env: TEnv) -> Self { + Self { env } + } + + pub fn get_closest_package_json( + &self, + url: &Url, + ) -> Result, ClosestPkgJsonError> { + let Ok(file_path) = deno_path_util::url_to_file_path(url) else { + return Ok(None); + }; + self.get_closest_package_json_from_path(&file_path) + } + + pub fn get_closest_package_json_from_path( + &self, + file_path: &Path, + ) -> Result, ClosestPkgJsonError> { + // we use this for deno compile using byonm because the script paths + // won't be in virtual file system, but the package.json paths will be + fn canonicalize_first_ancestor_exists( + dir_path: &Path, + env: &TEnv, + ) -> Result, std::io::Error> { + for ancestor in dir_path.ancestors() { + match env.realpath_sync(ancestor) { + Ok(dir_path) => return Ok(Some(dir_path)), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + // keep searching + } + Err(err) => return Err(err), + } + } + Ok(None) + } + + let parent_dir = file_path.parent().unwrap(); + let Some(start_dir) = canonicalize_first_ancestor_exists( + parent_dir, &self.env, + ) + .map_err(|source| CanonicalizingPkgJsonDirError { + dir_path: parent_dir.to_path_buf(), + source, + })? + else { + return Ok(None); + }; + let start_dir = strip_unc_prefix(start_dir); + for current_dir in start_dir.ancestors() { + let package_json_path = current_dir.join("package.json"); + if let Some(pkg_json) = self.load_package_json(&package_json_path)? { + return Ok(Some(pkg_json)); + } + } + + Ok(None) + } + + pub fn load_package_json( + &self, + path: &Path, + ) -> Result, PackageJsonLoadError> { + let result = PackageJson::load_from_path( + path, + self.env.pkg_json_fs(), + Some(&PackageJsonThreadLocalCache), + ); + match result { + Ok(pkg_json) => Ok(Some(pkg_json)), + Err(deno_package_json::PackageJsonLoadError::Io { source, .. }) + if source.kind() == ErrorKind::NotFound => + { + Ok(None) + } + Err(err) => Err(PackageJsonLoadError(err)), + } + } +} diff --git a/resolvers/node/path.rs b/resolvers/node/path.rs new file mode 100644 index 00000000000000..8c2d35fadf3fad --- /dev/null +++ b/resolvers/node/path.rs @@ -0,0 +1,81 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::path::Component; +use std::path::Path; +use std::path::PathBuf; + +/// Extension to path_clean::PathClean +pub trait PathClean { + fn clean(&self) -> T; +} + +impl PathClean for PathBuf { + fn clean(&self) -> PathBuf { + fn is_clean_path(path: &Path) -> bool { + let path = path.to_string_lossy(); + let mut current_index = 0; + while let Some(index) = path[current_index..].find("\\.") { + let trailing_index = index + current_index + 2; + let mut trailing_chars = path[trailing_index..].chars(); + match trailing_chars.next() { + Some('.') => match trailing_chars.next() { + Some('/') | Some('\\') | None => { + return false; + } + _ => {} + }, + Some('/') | Some('\\') => { + return false; + } + _ => {} + } + current_index = trailing_index; + } + true + } + + let path = path_clean::PathClean::clean(self); + if cfg!(windows) && !is_clean_path(&path) { + // temporary workaround because path_clean::PathClean::clean is + // not good enough on windows + let mut components = Vec::new(); + + for component in path.components() { + match component { + Component::CurDir => { + // skip + } + Component::ParentDir => { + let maybe_last_component = components.pop(); + if !matches!(maybe_last_component, Some(Component::Normal(_))) { + panic!("Error normalizing: {}", path.display()); + } + } + Component::Normal(_) | Component::RootDir | Component::Prefix(_) => { + components.push(component); + } + } + } + components.into_iter().collect::() + } else { + path + } + } +} + +#[cfg(test)] +mod test { + #[cfg(windows)] + #[test] + fn test_path_clean() { + use super::*; + + run_test("C:\\test\\./file.txt", "C:\\test\\file.txt"); + run_test("C:\\test\\../other/file.txt", "C:\\other\\file.txt"); + run_test("C:\\test\\../other\\file.txt", "C:\\other\\file.txt"); + + fn run_test(input: &str, expected: &str) { + assert_eq!(PathBuf::from(input).clean(), PathBuf::from(expected)); + } + } +} diff --git a/ext/node_resolver/resolution.rs b/resolvers/node/resolution.rs similarity index 81% rename from ext/node_resolver/resolution.rs rename to resolvers/node/resolution.rs index ad9dbb7100c092..5f87698cd6eeaf 100644 --- a/ext/node_resolver/resolution.rs +++ b/resolvers/node/resolution.rs @@ -6,16 +6,13 @@ use std::path::PathBuf; use anyhow::bail; use anyhow::Error as AnyError; -use deno_media_type::MediaType; -use deno_package_json::PackageJsonRc; +use deno_path_util::url_from_file_path; use serde_json::Map; use serde_json::Value; use url::Url; use crate::env::NodeResolverEnv; use crate::errors; -use crate::errors::CanonicalizingPkgJsonDirError; -use crate::errors::ClosestPkgJsonError; use crate::errors::DataUrlReferrerError; use crate::errors::FinalizeResolutionError; use crate::errors::InvalidModuleSpecifierError; @@ -30,7 +27,6 @@ use crate::errors::PackageExportsResolveError; use crate::errors::PackageImportNotDefinedError; use crate::errors::PackageImportsResolveError; use crate::errors::PackageImportsResolveErrorKind; -use crate::errors::PackageJsonLoadError; use crate::errors::PackagePathNotExportedError; use crate::errors::PackageResolveError; use crate::errors::PackageSubpathResolveError; @@ -40,16 +36,13 @@ use crate::errors::PackageTargetResolveError; use crate::errors::PackageTargetResolveErrorKind; use crate::errors::ResolveBinaryCommandsError; use crate::errors::ResolvePkgJsonBinExportError; -use crate::errors::ResolvePkgSubpathFromDenoModuleError; -use crate::errors::TypeScriptNotSupportedInNpmError; use crate::errors::TypesNotFoundError; use crate::errors::TypesNotFoundErrorData; use crate::errors::UnsupportedDirImportError; use crate::errors::UnsupportedEsmUrlSchemeError; -use crate::errors::UrlToNodeResolutionError; -use crate::path::strip_unc_prefix; -use crate::path::to_file_specifier; -use crate::NpmResolverRc; +use crate::npm::InNpmPackageCheckerRc; +use crate::NpmPackageFolderResolverRc; +use crate::PackageJsonResolverRc; use crate::PathClean; use deno_package_json::PackageJson; @@ -57,32 +50,43 @@ pub static DEFAULT_CONDITIONS: &[&str] = &["deno", "node", "import"]; pub static REQUIRE_CONDITIONS: &[&str] = &["require", "node"]; static TYPES_ONLY_CONDITIONS: &[&str] = &["types"]; -pub type NodeModuleKind = deno_package_json::NodeModuleKind; +fn conditions_from_resolution_mode( + resolution_mode: ResolutionMode, +) -> &'static [&'static str] { + match resolution_mode { + ResolutionMode::Import => DEFAULT_CONDITIONS, + ResolutionMode::Require => REQUIRE_CONDITIONS, + } +} -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum NodeResolutionMode { +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum ResolutionMode { + Import, + Require, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum NodeResolutionKind { Execution, Types, } -impl NodeResolutionMode { +impl NodeResolutionKind { pub fn is_types(&self) -> bool { - matches!(self, NodeResolutionMode::Types) + matches!(self, NodeResolutionKind::Types) } } #[derive(Debug)] pub enum NodeResolution { - Esm(Url), - CommonJs(Url), + Module(Url), BuiltIn(String), } impl NodeResolution { pub fn into_url(self) -> Url { match self { - Self::Esm(u) => u, - Self::CommonJs(u) => u, + Self::Module(u) => u, Self::BuiltIn(specifier) => { if specifier.starts_with("node:") { Url::parse(&specifier).unwrap() @@ -92,42 +96,6 @@ impl NodeResolution { } } } - - pub fn into_specifier_and_media_type( - resolution: Option, - ) -> (Url, MediaType) { - match resolution { - Some(NodeResolution::CommonJs(specifier)) => { - let media_type = MediaType::from_specifier(&specifier); - ( - specifier, - match media_type { - MediaType::JavaScript | MediaType::Jsx => MediaType::Cjs, - MediaType::TypeScript | MediaType::Tsx => MediaType::Cts, - MediaType::Dts => MediaType::Dcts, - _ => media_type, - }, - ) - } - Some(NodeResolution::Esm(specifier)) => { - let media_type = MediaType::from_specifier(&specifier); - ( - specifier, - match media_type { - MediaType::JavaScript | MediaType::Jsx => MediaType::Mjs, - MediaType::TypeScript | MediaType::Tsx => MediaType::Mts, - MediaType::Dts => MediaType::Dmts, - _ => media_type, - }, - ) - } - Some(resolution) => (resolution.into_url(), MediaType::Dts), - None => ( - Url::parse("internal:///missing_dependency.d.ts").unwrap(), - MediaType::Dts, - ), - } - } } #[allow(clippy::disallowed_types)] @@ -136,16 +104,28 @@ pub type NodeResolverRc = crate::sync::MaybeArc>; #[derive(Debug)] pub struct NodeResolver { env: TEnv, - npm_resolver: NpmResolverRc, + in_npm_pkg_checker: InNpmPackageCheckerRc, + npm_pkg_folder_resolver: NpmPackageFolderResolverRc, + pkg_json_resolver: PackageJsonResolverRc, } impl NodeResolver { - pub fn new(env: TEnv, npm_resolver: NpmResolverRc) -> Self { - Self { env, npm_resolver } + pub fn new( + env: TEnv, + in_npm_pkg_checker: InNpmPackageCheckerRc, + npm_pkg_folder_resolver: NpmPackageFolderResolverRc, + pkg_json_resolver: PackageJsonResolverRc, + ) -> Self { + Self { + env, + in_npm_pkg_checker, + npm_pkg_folder_resolver, + pkg_json_resolver, + } } pub fn in_npm_package(&self, specifier: &Url) -> bool { - self.npm_resolver.in_npm_package(specifier) + self.in_npm_pkg_checker.in_npm_package(specifier) } /// This function is an implementation of `defaultResolve` in @@ -154,8 +134,8 @@ impl NodeResolver { &self, specifier: &str, referrer: &Url, - referrer_kind: NodeModuleKind, - mode: NodeResolutionMode, + resolution_mode: ResolutionMode, + resolution_kind: NodeResolutionKind, ) -> Result { // Note: if we are here, then the referrer is an esm module // TODO(bartlomieju): skipped "policy" part as we don't plan to support it @@ -166,7 +146,7 @@ impl NodeResolver { if let Ok(url) = Url::parse(specifier) { if url.scheme() == "data" { - return Ok(NodeResolution::Esm(url)); + return Ok(NodeResolution::Module(url)); } if let Some(module_name) = @@ -191,28 +171,31 @@ impl NodeResolver { let url = referrer .join(specifier) .map_err(|source| DataUrlReferrerError { source })?; - return Ok(NodeResolution::Esm(url)); + return Ok(NodeResolution::Module(url)); } } let url = self.module_resolve( specifier, referrer, - referrer_kind, - // even though the referrer may be CJS, if we're here that means we're doing ESM resolution - DEFAULT_CONDITIONS, - mode, + resolution_mode, + conditions_from_resolution_mode(resolution_mode), + resolution_kind, )?; - let url = if mode.is_types() { + let url = if resolution_kind.is_types() { let file_path = to_file_path(&url); - self.path_to_declaration_url(&file_path, Some(referrer), referrer_kind)? + self.path_to_declaration_url( + &file_path, + Some(referrer), + resolution_mode, + )? } else { url }; let url = self.finalize_resolution(url, Some(referrer))?; - let resolve_response = self.url_to_node_resolution(url)?; + let resolve_response = NodeResolution::Module(url); // TODO(bartlomieju): skipped checking errors for commonJS resolution and // "preserveSymlinksMain"/"preserveSymlinks" options. Ok(resolve_response) @@ -222,12 +205,12 @@ impl NodeResolver { &self, specifier: &str, referrer: &Url, - referrer_kind: NodeModuleKind, + resolution_mode: ResolutionMode, conditions: &[&str], - mode: NodeResolutionMode, + resolution_kind: NodeResolutionKind, ) -> Result { if should_be_treated_as_relative_or_absolute_path(specifier) { - Ok(referrer.join(specifier).map_err(|err| { + Ok(node_join_url(referrer, specifier).map_err(|err| { NodeResolveRelativeJoinError { path: specifier.to_string(), base: referrer.clone(), @@ -236,16 +219,17 @@ impl NodeResolver { })?) } else if specifier.starts_with('#') { let pkg_config = self + .pkg_json_resolver .get_closest_package_json(referrer) .map_err(PackageImportsResolveErrorKind::ClosestPkgJson) .map_err(|err| PackageImportsResolveError(Box::new(err)))?; Ok(self.package_imports_resolve( specifier, Some(referrer), - referrer_kind, + resolution_mode, pkg_config.as_deref(), conditions, - mode, + resolution_kind, )?) } else if let Ok(resolved) = Url::parse(specifier) { Ok(resolved) @@ -253,9 +237,9 @@ impl NodeResolver { Ok(self.package_resolve( specifier, referrer, - referrer_kind, + resolution_mode, conditions, - mode, + resolution_kind, )?) } } @@ -331,9 +315,9 @@ impl NodeResolver { package_dir: &Path, package_subpath: Option<&str>, maybe_referrer: Option<&Url>, - mode: NodeResolutionMode, - ) -> Result { - let node_module_kind = NodeModuleKind::Esm; + resolution_mode: ResolutionMode, + resolution_kind: NodeResolutionKind, + ) -> Result { let package_subpath = package_subpath .map(|s| format!("./{s}")) .unwrap_or_else(|| ".".to_string()); @@ -341,14 +325,13 @@ impl NodeResolver { package_dir, &package_subpath, maybe_referrer, - node_module_kind, - DEFAULT_CONDITIONS, - mode, + resolution_mode, + conditions_from_resolution_mode(resolution_mode), + resolution_kind, )?; - let resolve_response = self.url_to_node_resolution(resolved_url)?; // TODO(bartlomieju): skipped checking errors for commonJS resolution and // "preserveSymlinksMain"/"preserveSymlinks" options. - Ok(resolve_response) + Ok(resolved_url) } pub fn resolve_binary_commands( @@ -356,7 +339,9 @@ impl NodeResolver { package_folder: &Path, ) -> Result, ResolveBinaryCommandsError> { let pkg_json_path = package_folder.join("package.json"); - let Some(package_json) = self.load_package_json(&pkg_json_path)? else { + let Some(package_json) = + self.pkg_json_resolver.load_package_json(&pkg_json_path)? + else { return Ok(Vec::new()); }; @@ -381,9 +366,11 @@ impl NodeResolver { &self, package_folder: &Path, sub_path: Option<&str>, - ) -> Result { + ) -> Result { let pkg_json_path = package_folder.join("package.json"); - let Some(package_json) = self.load_package_json(&pkg_json_path)? else { + let Some(package_json) = + self.pkg_json_resolver.load_package_json(&pkg_json_path)? + else { return Err(ResolvePkgJsonBinExportError::MissingPkgJson { pkg_json_path, }); @@ -394,39 +381,11 @@ impl NodeResolver { message: err.to_string(), } })?; - let url = to_file_specifier(&package_folder.join(bin_entry)); + let url = url_from_file_path(&package_folder.join(bin_entry)).unwrap(); - let resolve_response = self.url_to_node_resolution(url)?; // TODO(bartlomieju): skipped checking errors for commonJS resolution and // "preserveSymlinksMain"/"preserveSymlinks" options. - Ok(resolve_response) - } - - pub fn url_to_node_resolution( - &self, - url: Url, - ) -> Result { - let url_str = url.as_str().to_lowercase(); - if url_str.starts_with("http") || url_str.ends_with(".json") { - Ok(NodeResolution::Esm(url)) - } else if url_str.ends_with(".js") || url_str.ends_with(".d.ts") { - let maybe_package_config = self.get_closest_package_json(&url)?; - match maybe_package_config { - Some(c) if c.typ == "module" => Ok(NodeResolution::Esm(url)), - Some(_) => Ok(NodeResolution::CommonJs(url)), - None => Ok(NodeResolution::Esm(url)), - } - } else if url_str.ends_with(".mjs") || url_str.ends_with(".d.mts") { - Ok(NodeResolution::Esm(url)) - } else if url_str.ends_with(".ts") || url_str.ends_with(".mts") { - if self.in_npm_package(&url) { - Err(TypeScriptNotSupportedInNpmError { specifier: url }.into()) - } else { - Ok(NodeResolution::Esm(url)) - } - } else { - Ok(NodeResolution::CommonJs(url)) - } + Ok(url) } /// Checks if the resolved file has a corresponding declaration file. @@ -434,13 +393,13 @@ impl NodeResolver { &self, path: &Path, maybe_referrer: Option<&Url>, - referrer_kind: NodeModuleKind, + resolution_mode: ResolutionMode, ) -> Result { fn probe_extensions( fs: &TEnv, path: &Path, lowercase_path: &str, - referrer_kind: NodeModuleKind, + resolution_mode: ResolutionMode, ) -> Option { let mut searched_for_d_mts = false; let mut searched_for_d_cts = false; @@ -463,11 +422,11 @@ impl NodeResolver { return Some(dts_path); } - let specific_dts_path = match referrer_kind { - NodeModuleKind::Cjs if !searched_for_d_cts => { + let specific_dts_path = match resolution_mode { + ResolutionMode::Require if !searched_for_d_cts => { Some(with_known_extension(path, "d.cts")) } - NodeModuleKind::Esm if !searched_for_d_mts => { + ResolutionMode::Import if !searched_for_d_mts => { Some(with_known_extension(path, "d.mts")) } _ => None, // already searched above @@ -485,24 +444,21 @@ impl NodeResolver { || lowercase_path.ends_with(".d.cts") || lowercase_path.ends_with(".d.mts") { - return Ok(to_file_specifier(path)); + return Ok(url_from_file_path(path).unwrap()); } if let Some(path) = - probe_extensions(&self.env, path, &lowercase_path, referrer_kind) + probe_extensions(&self.env, path, &lowercase_path, resolution_mode) { - return Ok(to_file_specifier(&path)); + return Ok(url_from_file_path(&path).unwrap()); } if self.env.is_dir_sync(path) { let resolution_result = self.resolve_package_dir_subpath( path, /* sub path */ ".", maybe_referrer, - referrer_kind, - match referrer_kind { - NodeModuleKind::Esm => DEFAULT_CONDITIONS, - NodeModuleKind::Cjs => REQUIRE_CONDITIONS, - }, - NodeResolutionMode::Types, + resolution_mode, + conditions_from_resolution_mode(resolution_mode), + NodeResolutionKind::Types, ); if let Ok(resolution) = resolution_result { return Ok(resolution); @@ -512,17 +468,17 @@ impl NodeResolver { &self.env, &index_path, &index_path.to_string_lossy().to_lowercase(), - referrer_kind, + resolution_mode, ) { - return Ok(to_file_specifier(&path)); + return Ok(url_from_file_path(&path).unwrap()); } } // allow resolving .css files for types resolution if lowercase_path.ends_with(".css") { - return Ok(to_file_specifier(path)); + return Ok(url_from_file_path(path).unwrap()); } Err(TypesNotFoundError(Box::new(TypesNotFoundErrorData { - code_specifier: to_file_specifier(path), + code_specifier: url_from_file_path(path).unwrap(), maybe_referrer: maybe_referrer.cloned(), }))) } @@ -532,10 +488,10 @@ impl NodeResolver { &self, name: &str, maybe_referrer: Option<&Url>, - referrer_kind: NodeModuleKind, + resolution_mode: ResolutionMode, referrer_pkg_json: Option<&PackageJson>, conditions: &[&str], - mode: NodeResolutionMode, + resolution_kind: NodeResolutionKind, ) -> Result { if name == "#" || name.starts_with("#/") || name.ends_with('/') { let reason = "is not a valid internal imports specifier name"; @@ -561,11 +517,11 @@ impl NodeResolver { "", name, maybe_referrer, - referrer_kind, + resolution_mode, false, true, conditions, - mode, + resolution_kind, )?; if let Some(resolved) = maybe_resolved { return Ok(resolved); @@ -601,11 +557,11 @@ impl NodeResolver { best_match_subpath.unwrap(), best_match, maybe_referrer, - referrer_kind, + resolution_mode, true, true, conditions, - mode, + resolution_kind, )?; if let Some(resolved) = maybe_resolved { return Ok(resolved); @@ -633,11 +589,11 @@ impl NodeResolver { match_: &str, package_json_path: &Path, maybe_referrer: Option<&Url>, - referrer_kind: NodeModuleKind, + resolution_mode: ResolutionMode, pattern: bool, internal: bool, conditions: &[&str], - mode: NodeResolutionMode, + resolution_kind: NodeResolutionKind, ) -> Result { if !subpath.is_empty() && !pattern && !target.ends_with('/') { return Err( @@ -673,13 +629,14 @@ impl NodeResolver { } else { format!("{target}{subpath}") }; - let package_json_url = to_file_specifier(package_json_path); + let package_json_url = + url_from_file_path(package_json_path).unwrap(); let result = match self.package_resolve( &export_target, &package_json_url, - referrer_kind, + resolution_mode, conditions, - mode, + resolution_kind, ) { Ok(url) => Ok(url), Err(err) => match err.code() { @@ -700,8 +657,8 @@ impl NodeResolver { pkg_json_path: package_json_path.to_path_buf(), target: export_target.to_string(), maybe_referrer: maybe_referrer.map(ToOwned::to_owned), - referrer_kind, - mode, + resolution_mode, + resolution_kind, }, ) .into(), @@ -760,7 +717,7 @@ impl NodeResolver { ); } if subpath.is_empty() { - return Ok(to_file_specifier(&resolved_path)); + return Ok(url_from_file_path(&resolved_path).unwrap()); } if invalid_segment_re.is_match(subpath) { let request = if pattern { @@ -782,9 +739,11 @@ impl NodeResolver { let resolved_path_str = resolved_path.to_string_lossy(); let replaced = pattern_re .replace(&resolved_path_str, |_caps: ®ex::Captures| subpath); - return Ok(to_file_specifier(&PathBuf::from(replaced.to_string()))); + return Ok( + url_from_file_path(&PathBuf::from(replaced.to_string())).unwrap(), + ); } - Ok(to_file_specifier(&resolved_path.join(subpath).clean())) + Ok(url_from_file_path(&resolved_path.join(subpath).clean()).unwrap()) } #[allow(clippy::too_many_arguments)] @@ -795,11 +754,11 @@ impl NodeResolver { subpath: &str, package_subpath: &str, maybe_referrer: Option<&Url>, - referrer_kind: NodeModuleKind, + resolution_mode: ResolutionMode, pattern: bool, internal: bool, conditions: &[&str], - mode: NodeResolutionMode, + resolution_kind: NodeResolutionKind, ) -> Result, PackageTargetResolveError> { let result = self.resolve_package_target_inner( package_json_path, @@ -807,16 +766,16 @@ impl NodeResolver { subpath, package_subpath, maybe_referrer, - referrer_kind, + resolution_mode, pattern, internal, conditions, - mode, + resolution_kind, ); match result { Ok(maybe_resolved) => Ok(maybe_resolved), Err(err) => { - if mode.is_types() + if resolution_kind.is_types() && err.code() == NodeJsErrorCode::ERR_TYPES_NOT_FOUND && conditions != TYPES_ONLY_CONDITIONS { @@ -828,11 +787,11 @@ impl NodeResolver { subpath, package_subpath, maybe_referrer, - referrer_kind, + resolution_mode, pattern, internal, TYPES_ONLY_CONDITIONS, - mode, + resolution_kind, ) { return Ok(Some(resolved)); } @@ -851,11 +810,11 @@ impl NodeResolver { subpath: &str, package_subpath: &str, maybe_referrer: Option<&Url>, - referrer_kind: NodeModuleKind, + resolution_mode: ResolutionMode, pattern: bool, internal: bool, conditions: &[&str], - mode: NodeResolutionMode, + resolution_kind: NodeResolutionKind, ) -> Result, PackageTargetResolveError> { if let Some(target) = target.as_str() { let url = self.resolve_package_target_string( @@ -864,18 +823,18 @@ impl NodeResolver { package_subpath, package_json_path, maybe_referrer, - referrer_kind, + resolution_mode, pattern, internal, conditions, - mode, + resolution_kind, )?; - if mode.is_types() && url.scheme() == "file" { - let path = url.to_file_path().unwrap(); + if resolution_kind.is_types() && url.scheme() == "file" { + let path = deno_path_util::url_to_file_path(&url).unwrap(); return Ok(Some(self.path_to_declaration_url( &path, maybe_referrer, - referrer_kind, + resolution_mode, )?)); } else { return Ok(Some(url)); @@ -893,11 +852,11 @@ impl NodeResolver { subpath, package_subpath, maybe_referrer, - referrer_kind, + resolution_mode, pattern, internal, conditions, - mode, + resolution_kind, ); match resolved_result { @@ -931,7 +890,7 @@ impl NodeResolver { if key == "default" || conditions.contains(&key.as_str()) - || mode.is_types() && key.as_str() == "types" + || resolution_kind.is_types() && key.as_str() == "types" { let condition_target = target_obj.get(key).unwrap(); @@ -941,11 +900,11 @@ impl NodeResolver { subpath, package_subpath, maybe_referrer, - referrer_kind, + resolution_mode, pattern, internal, conditions, - mode, + resolution_kind, )?; match resolved { Some(resolved) => return Ok(Some(resolved)), @@ -978,9 +937,9 @@ impl NodeResolver { package_subpath: &str, package_exports: &Map, maybe_referrer: Option<&Url>, - referrer_kind: NodeModuleKind, + resolution_mode: ResolutionMode, conditions: &[&str], - mode: NodeResolutionMode, + resolution_kind: NodeResolutionKind, ) -> Result { if package_exports.contains_key(package_subpath) && package_subpath.find('*').is_none() @@ -993,11 +952,11 @@ impl NodeResolver { "", package_subpath, maybe_referrer, - referrer_kind, + resolution_mode, false, false, conditions, - mode, + resolution_kind, )?; return match resolved { Some(resolved) => Ok(resolved), @@ -1006,7 +965,7 @@ impl NodeResolver { pkg_json_path: package_json_path.to_path_buf(), subpath: package_subpath.to_string(), maybe_referrer: maybe_referrer.map(ToOwned::to_owned), - mode, + resolution_kind, } .into(), ), @@ -1055,11 +1014,11 @@ impl NodeResolver { &best_match_subpath.unwrap(), best_match, maybe_referrer, - referrer_kind, + resolution_mode, true, false, conditions, - mode, + resolution_kind, )?; if let Some(resolved) = maybe_resolved { return Ok(resolved); @@ -1069,7 +1028,7 @@ impl NodeResolver { pkg_json_path: package_json_path.to_path_buf(), subpath: package_subpath.to_string(), maybe_referrer: maybe_referrer.map(ToOwned::to_owned), - mode, + resolution_kind, } .into(), ); @@ -1081,7 +1040,7 @@ impl NodeResolver { pkg_json_path: package_json_path.to_path_buf(), subpath: package_subpath.to_string(), maybe_referrer: maybe_referrer.map(ToOwned::to_owned), - mode, + resolution_kind, } .into(), ) @@ -1091,14 +1050,16 @@ impl NodeResolver { &self, specifier: &str, referrer: &Url, - referrer_kind: NodeModuleKind, + resolution_mode: ResolutionMode, conditions: &[&str], - mode: NodeResolutionMode, + resolution_kind: NodeResolutionKind, ) -> Result { let (package_name, package_subpath, _is_scoped) = parse_npm_pkg_name(specifier, referrer)?; - if let Some(package_config) = self.get_closest_package_json(referrer)? { + if let Some(package_config) = + self.pkg_json_resolver.get_closest_package_json(referrer)? + { // ResolveSelf if package_config.name.as_ref() == Some(&package_name) { if let Some(exports) = &package_config.exports { @@ -1108,9 +1069,9 @@ impl NodeResolver { &package_subpath, exports, Some(referrer), - referrer_kind, + resolution_mode, conditions, - mode, + resolution_kind, ) .map_err(|err| err.into()); } @@ -1121,9 +1082,9 @@ impl NodeResolver { &package_name, &package_subpath, referrer, - referrer_kind, + resolution_mode, conditions, - mode, + resolution_kind, ) } @@ -1133,28 +1094,28 @@ impl NodeResolver { package_name: &str, package_subpath: &str, referrer: &Url, - referrer_kind: NodeModuleKind, + resolution_mode: ResolutionMode, conditions: &[&str], - mode: NodeResolutionMode, + resolution_kind: NodeResolutionKind, ) -> Result { let result = self.resolve_package_subpath_for_package_inner( package_name, package_subpath, referrer, - referrer_kind, + resolution_mode, conditions, - mode, + resolution_kind, ); - if mode.is_types() && !matches!(result, Ok(Url { .. })) { + if resolution_kind.is_types() && !matches!(result, Ok(Url { .. })) { // try to resolve with the @types package let package_name = types_package_name(package_name); if let Ok(result) = self.resolve_package_subpath_for_package_inner( &package_name, package_subpath, referrer, - referrer_kind, + resolution_mode, conditions, - mode, + resolution_kind, ) { return Ok(result); } @@ -1168,12 +1129,12 @@ impl NodeResolver { package_name: &str, package_subpath: &str, referrer: &Url, - referrer_kind: NodeModuleKind, + resolution_mode: ResolutionMode, conditions: &[&str], - mode: NodeResolutionMode, + resolution_kind: NodeResolutionKind, ) -> Result { let package_dir_path = self - .npm_resolver + .npm_pkg_folder_resolver .resolve_package_folder_from_package(package_name, referrer)?; // todo: error with this instead when can't find package @@ -1195,9 +1156,9 @@ impl NodeResolver { &package_dir_path, package_subpath, Some(referrer), - referrer_kind, + resolution_mode, conditions, - mode, + resolution_kind, ) .map_err(|err| err.into()) } @@ -1208,27 +1169,30 @@ impl NodeResolver { package_dir_path: &Path, package_subpath: &str, maybe_referrer: Option<&Url>, - referrer_kind: NodeModuleKind, + resolution_mode: ResolutionMode, conditions: &[&str], - mode: NodeResolutionMode, + resolution_kind: NodeResolutionKind, ) -> Result { let package_json_path = package_dir_path.join("package.json"); - match self.load_package_json(&package_json_path)? { + match self + .pkg_json_resolver + .load_package_json(&package_json_path)? + { Some(pkg_json) => self.resolve_package_subpath( &pkg_json, package_subpath, maybe_referrer, - referrer_kind, + resolution_mode, conditions, - mode, + resolution_kind, ), None => self .resolve_package_subpath_no_pkg_json( package_dir_path, package_subpath, maybe_referrer, - referrer_kind, - mode, + resolution_mode, + resolution_kind, ) .map_err(|err| { PackageSubpathResolveErrorKind::LegacyResolve(err).into() @@ -1242,9 +1206,9 @@ impl NodeResolver { package_json: &PackageJson, package_subpath: &str, referrer: Option<&Url>, - referrer_kind: NodeModuleKind, + resolution_mode: ResolutionMode, conditions: &[&str], - mode: NodeResolutionMode, + resolution_kind: NodeResolutionKind, ) -> Result { if let Some(exports) = &package_json.exports { let result = self.package_exports_resolve( @@ -1252,16 +1216,21 @@ impl NodeResolver { package_subpath, exports, referrer, - referrer_kind, + resolution_mode, conditions, - mode, + resolution_kind, ); match result { Ok(found) => return Ok(found), Err(exports_err) => { - if mode.is_types() && package_subpath == "." { + if resolution_kind.is_types() && package_subpath == "." { return self - .legacy_main_resolve(package_json, referrer, referrer_kind, mode) + .legacy_main_resolve( + package_json, + referrer, + resolution_mode, + resolution_kind, + ) .map_err(|err| { PackageSubpathResolveErrorKind::LegacyResolve(err).into() }); @@ -1275,7 +1244,12 @@ impl NodeResolver { if package_subpath == "." { return self - .legacy_main_resolve(package_json, referrer, referrer_kind, mode) + .legacy_main_resolve( + package_json, + referrer, + resolution_mode, + resolution_kind, + ) .map_err(|err| { PackageSubpathResolveErrorKind::LegacyResolve(err).into() }); @@ -1286,8 +1260,8 @@ impl NodeResolver { package_json.path.parent().unwrap(), package_subpath, referrer, - referrer_kind, - mode, + resolution_mode, + resolution_kind, ) .map_err(|err| { PackageSubpathResolveErrorKind::LegacyResolve(err.into()).into() @@ -1299,15 +1273,15 @@ impl NodeResolver { directory: &Path, package_subpath: &str, referrer: Option<&Url>, - referrer_kind: NodeModuleKind, - mode: NodeResolutionMode, + resolution_mode: ResolutionMode, + resolution_kind: NodeResolutionKind, ) -> Result { assert_ne!(package_subpath, "."); let file_path = directory.join(package_subpath); - if mode.is_types() { - Ok(self.path_to_declaration_url(&file_path, referrer, referrer_kind)?) + if resolution_kind.is_types() { + Ok(self.path_to_declaration_url(&file_path, referrer, resolution_mode)?) } else { - Ok(to_file_specifier(&file_path)) + Ok(url_from_file_path(&file_path).unwrap()) } } @@ -1316,107 +1290,52 @@ impl NodeResolver { directory: &Path, package_subpath: &str, maybe_referrer: Option<&Url>, - referrer_kind: NodeModuleKind, - mode: NodeResolutionMode, + resolution_mode: ResolutionMode, + resolution_kind: NodeResolutionKind, ) -> Result { if package_subpath == "." { - self.legacy_index_resolve(directory, maybe_referrer, referrer_kind, mode) + self.legacy_index_resolve( + directory, + maybe_referrer, + resolution_mode, + resolution_kind, + ) } else { self .resolve_subpath_exact( directory, package_subpath, maybe_referrer, - referrer_kind, - mode, + resolution_mode, + resolution_kind, ) .map_err(|err| err.into()) } } - pub fn get_closest_package_json( - &self, - url: &Url, - ) -> Result, ClosestPkgJsonError> { - let Ok(file_path) = url.to_file_path() else { - return Ok(None); - }; - self.get_closest_package_json_from_path(&file_path) - } - - pub fn get_closest_package_json_from_path( - &self, - file_path: &Path, - ) -> Result, ClosestPkgJsonError> { - // we use this for deno compile using byonm because the script paths - // won't be in virtual file system, but the package.json paths will be - fn canonicalize_first_ancestor_exists( - dir_path: &Path, - env: &dyn NodeResolverEnv, - ) -> Result, std::io::Error> { - for ancestor in dir_path.ancestors() { - match env.realpath_sync(ancestor) { - Ok(dir_path) => return Ok(Some(dir_path)), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => { - // keep searching - } - Err(err) => return Err(err), - } - } - Ok(None) - } - - let parent_dir = file_path.parent().unwrap(); - let Some(start_dir) = canonicalize_first_ancestor_exists( - parent_dir, &self.env, - ) - .map_err(|source| CanonicalizingPkgJsonDirError { - dir_path: parent_dir.to_path_buf(), - source, - })? - else { - return Ok(None); - }; - let start_dir = strip_unc_prefix(start_dir); - for current_dir in start_dir.ancestors() { - let package_json_path = current_dir.join("package.json"); - if let Some(pkg_json) = self.load_package_json(&package_json_path)? { - return Ok(Some(pkg_json)); - } - } - - Ok(None) - } - - pub fn load_package_json( - &self, - package_json_path: &Path, - ) -> Result, PackageJsonLoadError> { - crate::package_json::load_pkg_json( - self.env.pkg_json_fs(), - package_json_path, - ) - } - pub(super) fn legacy_main_resolve( &self, package_json: &PackageJson, maybe_referrer: Option<&Url>, - referrer_kind: NodeModuleKind, - mode: NodeResolutionMode, + resolution_mode: ResolutionMode, + resolution_kind: NodeResolutionKind, ) -> Result { - let maybe_main = if mode.is_types() { + let pkg_json_kind = match resolution_mode { + ResolutionMode::Require => deno_package_json::NodeModuleKind::Cjs, + ResolutionMode::Import => deno_package_json::NodeModuleKind::Esm, + }; + let maybe_main = if resolution_kind.is_types() { match package_json.types.as_ref() { Some(types) => Some(types.as_str()), None => { // fallback to checking the main entrypoint for // a corresponding declaration file - if let Some(main) = package_json.main(referrer_kind) { + if let Some(main) = package_json.main(pkg_json_kind) { let main = package_json.path.parent().unwrap().join(main).clean(); let decl_url_result = self.path_to_declaration_url( &main, maybe_referrer, - referrer_kind, + resolution_mode, ); // don't surface errors, fallback to checking the index now if let Ok(url) = decl_url_result { @@ -1427,22 +1346,22 @@ impl NodeResolver { } } } else { - package_json.main(referrer_kind) + package_json.main(pkg_json_kind) }; if let Some(main) = maybe_main { let guess = package_json.path.parent().unwrap().join(main).clean(); if self.env.is_file_sync(&guess) { - return Ok(to_file_specifier(&guess)); + return Ok(url_from_file_path(&guess).unwrap()); } // todo(dsherret): investigate exactly how node and typescript handles this - let endings = if mode.is_types() { - match referrer_kind { - NodeModuleKind::Cjs => { + let endings = if resolution_kind.is_types() { + match resolution_mode { + ResolutionMode::Require => { vec![".d.ts", ".d.cts", "/index.d.ts", "/index.d.cts"] } - NodeModuleKind::Esm => vec![ + ResolutionMode::Import => vec![ ".d.ts", ".d.mts", "/index.d.ts", @@ -1463,7 +1382,7 @@ impl NodeResolver { .clean(); if self.env.is_file_sync(&guess) { // TODO(bartlomieju): emitLegacyIndexDeprecation() - return Ok(to_file_specifier(&guess)); + return Ok(url_from_file_path(&guess).unwrap()); } } } @@ -1471,8 +1390,8 @@ impl NodeResolver { self.legacy_index_resolve( package_json.path.parent().unwrap(), maybe_referrer, - referrer_kind, - mode, + resolution_mode, + resolution_kind, ) } @@ -1480,14 +1399,16 @@ impl NodeResolver { &self, directory: &Path, maybe_referrer: Option<&Url>, - referrer_kind: NodeModuleKind, - mode: NodeResolutionMode, + resolution_mode: ResolutionMode, + resolution_kind: NodeResolutionKind, ) -> Result { - let index_file_names = if mode.is_types() { + let index_file_names = if resolution_kind.is_types() { // todo(dsherret): investigate exactly how typescript does this - match referrer_kind { - NodeModuleKind::Cjs => vec!["index.d.ts", "index.d.cts"], - NodeModuleKind::Esm => vec!["index.d.ts", "index.d.mts", "index.d.cts"], + match resolution_mode { + ResolutionMode::Require => vec!["index.d.ts", "index.d.cts"], + ResolutionMode::Import => { + vec!["index.d.ts", "index.d.mts", "index.d.cts"] + } } } else { vec!["index.js"] @@ -1496,14 +1417,15 @@ impl NodeResolver { let guess = directory.join(index_file_name).clean(); if self.env.is_file_sync(&guess) { // TODO(bartlomieju): emitLegacyIndexDeprecation() - return Ok(to_file_specifier(&guess)); + return Ok(url_from_file_path(&guess).unwrap()); } } - if mode.is_types() { + if resolution_kind.is_types() { Err( TypesNotFoundError(Box::new(TypesNotFoundErrorData { - code_specifier: to_file_specifier(&directory.join("index.js")), + code_specifier: url_from_file_path(&directory.join("index.js")) + .unwrap(), maybe_referrer: maybe_referrer.cloned(), })) .into(), @@ -1511,7 +1433,7 @@ impl NodeResolver { } else { Err( ModuleNotFoundError { - specifier: to_file_specifier(&directory.join("index.js")), + specifier: url_from_file_path(&directory.join("index.js")).unwrap(), typ: "module", maybe_referrer: maybe_referrer.cloned(), } @@ -1519,6 +1441,25 @@ impl NodeResolver { ) } } + + /// Resolves a specifier that is pointing into a node_modules folder by canonicalizing it. + /// + /// Returns `None` when the specifier is not in a node_modules folder. + pub fn handle_if_in_node_modules(&self, specifier: &Url) -> Option { + // skip canonicalizing if we definitely know it's unnecessary + if specifier.scheme() == "file" + && specifier.path().contains("/node_modules/") + { + // Specifiers in the node_modules directory are canonicalized + // so canoncalize then check if it's in the node_modules directory. + let specifier = resolve_specifier_into_node_modules(specifier, &|path| { + self.env.realpath_sync(path) + }); + return Some(specifier); + } + + None + } } fn resolve_bin_entry_value<'a>( @@ -1611,9 +1552,7 @@ fn resolve_bin_entry_value<'a>( } fn to_file_path(url: &Url) -> PathBuf { - url - .to_file_path() - .unwrap_or_else(|_| panic!("Provided URL was not file:// URL: {url}")) + deno_path_util::url_to_file_path(url).unwrap() } fn to_file_path_string(url: &Url) -> String { @@ -1692,7 +1631,7 @@ fn with_known_extension(path: &Path, ext: &str) -> PathBuf { } fn to_specifier_display_string(url: &Url) -> String { - if let Ok(path) = url.to_file_path() { + if let Ok(path) = deno_path_util::url_to_file_path(url) { path.display().to_string() } else { url.to_string() @@ -1769,6 +1708,28 @@ pub fn parse_npm_pkg_name( Ok((package_name, package_subpath, is_scoped)) } +/// Resolves a specifier that is pointing into a node_modules folder. +/// +/// Note: This should be called whenever getting the specifier from +/// a Module::External(module) reference because that module might +/// not be fully resolved at the time deno_graph is analyzing it +/// because the node_modules folder might not exist at that time. +pub fn resolve_specifier_into_node_modules( + specifier: &Url, + canonicalize: &impl Fn(&Path) -> std::io::Result, +) -> Url { + deno_path_util::url_to_file_path(specifier) + .ok() + // this path might not exist at the time the graph is being created + // because the node_modules folder might not yet exist + .and_then(|path| { + deno_path_util::canonicalize_path_maybe_not_exists(&path, canonicalize) + .ok() + }) + .and_then(|path| deno_path_util::url_from_file_path(&path).ok()) + .unwrap_or_else(|| specifier.clone()) +} + fn pattern_key_compare(a: &str, b: &str) -> i32 { let a_pattern_index = a.find('*'); let b_pattern_index = b.find('*'); @@ -1831,6 +1792,17 @@ fn get_module_name_from_builtin_node_module_specifier( Some(specifier) } +/// Node is more lenient joining paths than the url crate is, +/// so this function handles that. +fn node_join_url(url: &Url, path: &str) -> Result { + if let Some(suffix) = path.strip_prefix(".//") { + // specifier had two leading slashes + url.join(&format!("./{}", suffix)) + } else { + url.join(path) + } +} + #[cfg(test)] mod tests { use serde_json::json; diff --git a/ext/node_resolver/sync.rs b/resolvers/node/sync.rs similarity index 100% rename from ext/node_resolver/sync.rs rename to resolvers/node/sync.rs diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 6d24781cfa28ad..e4c0e188de406f 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_runtime" -version = "0.177.0" +version = "0.189.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -88,7 +88,9 @@ deno_kv.workspace = true deno_napi.workspace = true deno_net.workspace = true deno_node.workspace = true +deno_path_util.workspace = true deno_permissions.workspace = true +deno_telemetry.workspace = true deno_terminal.workspace = true deno_tls.workspace = true deno_url.workspace = true @@ -99,6 +101,7 @@ deno_websocket.workspace = true deno_webstorage.workspace = true node_resolver = { workspace = true, features = ["sync"] } +color-print.workspace = true dlopen2.workspace = true encoding_rs.workspace = true fastwebsockets.workspace = true @@ -115,9 +118,12 @@ once_cell.workspace = true percent-encoding.workspace = true regex.workspace = true rustyline = { workspace = true, features = ["custom-bindings"] } +same-file = "1.0.6" serde.workspace = true signal-hook = "0.3.17" signal-hook-registry = "1.4.0" +tempfile.workspace = true +thiserror.workspace = true tokio.workspace = true tokio-metrics.workspace = true twox-hash.workspace = true diff --git a/runtime/clippy.toml b/runtime/clippy.toml index 53676a90e6f2b2..79e6bbd08349fe 100644 --- a/runtime/clippy.toml +++ b/runtime/clippy.toml @@ -42,4 +42,5 @@ disallowed-methods = [ { path = "std::fs::write", reason = "File system operations should be done using FileSystem trait" }, { path = "std::path::Path::canonicalize", reason = "File system operations should be done using FileSystem trait" }, { path = "std::path::Path::exists", reason = "File system operations should be done using FileSystem trait" }, + { path = "std::process::exit", reason = "use deno_runtime::exit instead" }, ] diff --git a/runtime/code_cache.rs b/runtime/code_cache.rs index 2a56543a4127f8..b4a7ce188f5cdf 100644 --- a/runtime/code_cache.rs +++ b/runtime/code_cache.rs @@ -2,20 +2,12 @@ use deno_core::ModuleSpecifier; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum CodeCacheType { EsModule, Script, } -impl CodeCacheType { - pub fn as_str(&self) -> &str { - match self { - Self::EsModule => "esmodule", - Self::Script => "script", - } - } -} - pub trait CodeCache: Send + Sync { fn get_sync( &self, @@ -23,6 +15,7 @@ pub trait CodeCache: Send + Sync { code_cache_type: CodeCacheType, source_hash: u64, ) -> Option>; + fn set_sync( &self, specifier: ModuleSpecifier, diff --git a/runtime/errors.rs b/runtime/errors.rs index 694402773e5f99..4268fbd502df13 100644 --- a/runtime/errors.rs +++ b/runtime/errors.rs @@ -9,15 +9,118 @@ //! Diagnostics are compile-time type errors, whereas JsErrors are runtime //! exceptions. +use crate::ops::fs_events::FsEventsError; +use crate::ops::http::HttpStartError; +use crate::ops::os::OsError; +use crate::ops::permissions::PermissionError; +use crate::ops::process::CheckRunPermissionError; +use crate::ops::process::ProcessError; +use crate::ops::signal::SignalError; +use crate::ops::tty::TtyError; +use crate::ops::web_worker::SyncFetchError; +use crate::ops::worker_host::CreateWorkerError; +use deno_broadcast_channel::BroadcastChannelError; +use deno_cache::CacheError; +use deno_canvas::CanvasError; use deno_core::error::AnyError; use deno_core::serde_json; use deno_core::url; use deno_core::ModuleResolutionError; +use deno_cron::CronError; +use deno_crypto::DecryptError; +use deno_crypto::EncryptError; +use deno_crypto::ExportKeyError; +use deno_crypto::GenerateKeyError; +use deno_crypto::ImportKeyError; +use deno_fetch::FetchError; +use deno_fetch::HttpClientCreateError; +use deno_ffi::CallError; +use deno_ffi::CallbackError; +use deno_ffi::DlfcnError; +use deno_ffi::IRError; +use deno_ffi::ReprError; +use deno_ffi::StaticError; +use deno_fs::FsOpsError; +use deno_fs::FsOpsErrorKind; +use deno_http::HttpError; +use deno_http::HttpNextError; +use deno_http::WebSocketUpgradeError; +use deno_io::fs::FsError; +use deno_kv::KvCheckError; +use deno_kv::KvError; +use deno_kv::KvErrorKind; +use deno_kv::KvMutationError; +use deno_napi::NApiError; +use deno_net::ops::NetError; +use deno_permissions::ChildPermissionError; +use deno_permissions::NetDescriptorFromUrlParseError; +use deno_permissions::PathResolveError; +use deno_permissions::PermissionCheckError; +use deno_permissions::RunDescriptorParseError; +use deno_permissions::SysDescriptorParseError; +use deno_tls::TlsError; +use deno_web::BlobError; +use deno_web::CompressionError; +use deno_web::MessagePortError; +use deno_web::StreamResourceError; +use deno_web::WebError; +use deno_websocket::HandshakeError; +use deno_websocket::WebsocketError; +use deno_webstorage::WebStorageError; +use rustyline::error::ReadlineError; use std::env; use std::error::Error; use std::io; use std::sync::Arc; +fn get_run_descriptor_parse_error(e: &RunDescriptorParseError) -> &'static str { + match e { + RunDescriptorParseError::Which(_) => "Error", + RunDescriptorParseError::PathResolve(e) => get_path_resolve_error(e), + RunDescriptorParseError::EmptyRunQuery => "Error", + } +} + +fn get_sys_descriptor_parse_error(e: &SysDescriptorParseError) -> &'static str { + match e { + SysDescriptorParseError::InvalidKind(_) => "TypeError", + SysDescriptorParseError::Empty => "Error", + } +} + +fn get_path_resolve_error(e: &PathResolveError) -> &'static str { + match e { + PathResolveError::CwdResolve(e) => get_io_error_class(e), + PathResolveError::EmptyPath => "Error", + } +} + +fn get_permission_error_class(e: &PermissionError) -> &'static str { + match e { + PermissionError::InvalidPermissionName(_) => "ReferenceError", + PermissionError::PathResolve(e) => get_path_resolve_error(e), + PermissionError::NetDescriptorParse(_) => "URIError", + PermissionError::SysDescriptorParse(e) => get_sys_descriptor_parse_error(e), + PermissionError::RunDescriptorParse(e) => get_run_descriptor_parse_error(e), + } +} + +fn get_permission_check_error_class(e: &PermissionCheckError) -> &'static str { + match e { + PermissionCheckError::PermissionDenied(_) => "NotCapable", + PermissionCheckError::InvalidFilePath(_) => "URIError", + PermissionCheckError::NetDescriptorForUrlParse(e) => match e { + NetDescriptorFromUrlParseError::MissingHost(_) => "TypeError", + NetDescriptorFromUrlParseError::Host(_) => "URIError", + }, + PermissionCheckError::SysDescriptorParse(e) => { + get_sys_descriptor_parse_error(e) + } + PermissionCheckError::PathResolve(e) => get_path_resolve_error(e), + PermissionCheckError::HostParse(_) => "URIError", + } +} + fn get_dlopen_error_class(error: &dlopen2::Error) -> &'static str { use dlopen2::Error::*; match error { @@ -153,12 +256,1624 @@ pub fn get_nix_error_class(error: &nix::Error) -> &'static str { } } +fn get_webgpu_error_class(e: &deno_webgpu::InitError) -> &'static str { + match e { + deno_webgpu::InitError::Resource(e) => { + get_error_class_name(e).unwrap_or("Error") + } + deno_webgpu::InitError::InvalidAdapter(_) => "Error", + deno_webgpu::InitError::RequestDevice(_) => "DOMExceptionOperationError", + deno_webgpu::InitError::InvalidDevice(_) => "Error", + } +} + +fn get_webgpu_buffer_error_class( + e: &deno_webgpu::buffer::BufferError, +) -> &'static str { + match e { + deno_webgpu::buffer::BufferError::Resource(e) => { + get_error_class_name(e).unwrap_or("Error") + } + deno_webgpu::buffer::BufferError::InvalidUsage => "TypeError", + deno_webgpu::buffer::BufferError::Access(_) => "DOMExceptionOperationError", + } +} + +fn get_webgpu_bundle_error_class( + e: &deno_webgpu::bundle::BundleError, +) -> &'static str { + match e { + deno_webgpu::bundle::BundleError::Resource(e) => { + get_error_class_name(e).unwrap_or("Error") + } + deno_webgpu::bundle::BundleError::InvalidSize => "TypeError", + } +} + +fn get_webgpu_byow_error_class( + e: &deno_webgpu::byow::ByowError, +) -> &'static str { + match e { + deno_webgpu::byow::ByowError::WebGPUNotInitiated => "TypeError", + deno_webgpu::byow::ByowError::InvalidParameters => "TypeError", + deno_webgpu::byow::ByowError::CreateSurface(_) => "Error", + deno_webgpu::byow::ByowError::InvalidSystem => "TypeError", + #[cfg(any( + target_os = "windows", + target_os = "linux", + target_os = "freebsd", + target_os = "openbsd" + ))] + deno_webgpu::byow::ByowError::NullWindow => "TypeError", + #[cfg(any( + target_os = "linux", + target_os = "freebsd", + target_os = "openbsd" + ))] + deno_webgpu::byow::ByowError::NullDisplay => "TypeError", + #[cfg(target_os = "macos")] + deno_webgpu::byow::ByowError::NSViewDisplay => "TypeError", + } +} + +fn get_webgpu_render_pass_error_class( + e: &deno_webgpu::render_pass::RenderPassError, +) -> &'static str { + match e { + deno_webgpu::render_pass::RenderPassError::Resource(e) => { + get_error_class_name(e).unwrap_or("Error") + } + deno_webgpu::render_pass::RenderPassError::InvalidSize => "TypeError", + } +} + +fn get_webgpu_surface_error_class( + e: &deno_webgpu::surface::SurfaceError, +) -> &'static str { + match e { + deno_webgpu::surface::SurfaceError::Resource(e) => { + get_error_class_name(e).unwrap_or("Error") + } + deno_webgpu::surface::SurfaceError::Surface(_) => "Error", + deno_webgpu::surface::SurfaceError::InvalidStatus => "Error", + } +} + +fn get_crypto_decrypt_error_class(e: &DecryptError) -> &'static str { + match e { + DecryptError::General(e) => get_crypto_shared_error_class(e), + DecryptError::Pkcs1(_) => "Error", + DecryptError::Failed => "DOMExceptionOperationError", + DecryptError::InvalidLength => "TypeError", + DecryptError::InvalidCounterLength => "TypeError", + DecryptError::InvalidTagLength => "TypeError", + DecryptError::InvalidKeyOrIv => "DOMExceptionOperationError", + DecryptError::TooMuchData => "DOMExceptionOperationError", + DecryptError::InvalidIvLength => "TypeError", + DecryptError::Rsa(_) => "DOMExceptionOperationError", + } +} + +fn get_crypto_encrypt_error_class(e: &EncryptError) -> &'static str { + match e { + EncryptError::General(e) => get_crypto_shared_error_class(e), + EncryptError::InvalidKeyOrIv => "DOMExceptionOperationError", + EncryptError::Failed => "DOMExceptionOperationError", + EncryptError::InvalidLength => "TypeError", + EncryptError::InvalidIvLength => "TypeError", + EncryptError::InvalidCounterLength => "TypeError", + EncryptError::TooMuchData => "DOMExceptionOperationError", + } +} + +fn get_crypto_shared_error_class(e: &deno_crypto::SharedError) -> &'static str { + match e { + deno_crypto::SharedError::ExpectedValidPrivateKey => "TypeError", + deno_crypto::SharedError::ExpectedValidPublicKey => "TypeError", + deno_crypto::SharedError::ExpectedValidPrivateECKey => "TypeError", + deno_crypto::SharedError::ExpectedValidPublicECKey => "TypeError", + deno_crypto::SharedError::ExpectedPrivateKey => "TypeError", + deno_crypto::SharedError::ExpectedPublicKey => "TypeError", + deno_crypto::SharedError::ExpectedSecretKey => "TypeError", + deno_crypto::SharedError::FailedDecodePrivateKey => { + "DOMExceptionOperationError" + } + deno_crypto::SharedError::FailedDecodePublicKey => { + "DOMExceptionOperationError" + } + deno_crypto::SharedError::UnsupportedFormat => { + "DOMExceptionNotSupportedError" + } + } +} + +fn get_crypto_ed25519_error_class( + e: &deno_crypto::Ed25519Error, +) -> &'static str { + match e { + deno_crypto::Ed25519Error::FailedExport => "DOMExceptionOperationError", + deno_crypto::Ed25519Error::Der(_) => "Error", + deno_crypto::Ed25519Error::KeyRejected(_) => "Error", + } +} + +fn get_crypto_export_key_error_class(e: &ExportKeyError) -> &'static str { + match e { + ExportKeyError::General(e) => get_crypto_shared_error_class(e), + ExportKeyError::Der(_) => "Error", + ExportKeyError::UnsupportedNamedCurve => "DOMExceptionNotSupportedError", + } +} + +fn get_crypto_generate_key_error_class(e: &GenerateKeyError) -> &'static str { + match e { + GenerateKeyError::General(e) => get_crypto_shared_error_class(e), + GenerateKeyError::BadPublicExponent => "DOMExceptionOperationError", + GenerateKeyError::InvalidHMACKeyLength => "DOMExceptionOperationError", + GenerateKeyError::FailedRSAKeySerialization => "DOMExceptionOperationError", + GenerateKeyError::InvalidAESKeyLength => "DOMExceptionOperationError", + GenerateKeyError::FailedRSAKeyGeneration => "DOMExceptionOperationError", + GenerateKeyError::FailedECKeyGeneration => "DOMExceptionOperationError", + GenerateKeyError::FailedKeyGeneration => "DOMExceptionOperationError", + } +} + +fn get_crypto_import_key_error_class(e: &ImportKeyError) -> &'static str { + match e { + ImportKeyError::General(e) => get_crypto_shared_error_class(e), + ImportKeyError::InvalidModulus => "DOMExceptionDataError", + ImportKeyError::InvalidPublicExponent => "DOMExceptionDataError", + ImportKeyError::InvalidPrivateExponent => "DOMExceptionDataError", + ImportKeyError::InvalidFirstPrimeFactor => "DOMExceptionDataError", + ImportKeyError::InvalidSecondPrimeFactor => "DOMExceptionDataError", + ImportKeyError::InvalidFirstCRTExponent => "DOMExceptionDataError", + ImportKeyError::InvalidSecondCRTExponent => "DOMExceptionDataError", + ImportKeyError::InvalidCRTCoefficient => "DOMExceptionDataError", + ImportKeyError::InvalidB64Coordinate => "DOMExceptionDataError", + ImportKeyError::InvalidRSAPublicKey => "DOMExceptionDataError", + ImportKeyError::InvalidRSAPrivateKey => "DOMExceptionDataError", + ImportKeyError::UnsupportedAlgorithm => "DOMExceptionDataError", + ImportKeyError::PublicKeyTooLong => "DOMExceptionDataError", + ImportKeyError::PrivateKeyTooLong => "DOMExceptionDataError", + ImportKeyError::InvalidP256ECPoint => "DOMExceptionDataError", + ImportKeyError::InvalidP384ECPoint => "DOMExceptionDataError", + ImportKeyError::InvalidP521ECPoint => "DOMExceptionDataError", + ImportKeyError::UnsupportedNamedCurve => "DOMExceptionDataError", + ImportKeyError::CurveMismatch => "DOMExceptionDataError", + ImportKeyError::InvalidKeyData => "DOMExceptionDataError", + ImportKeyError::InvalidJWKPrivateKey => "DOMExceptionDataError", + ImportKeyError::EllipticCurve(_) => "DOMExceptionDataError", + ImportKeyError::ExpectedValidPkcs8Data => "DOMExceptionDataError", + ImportKeyError::MalformedParameters => "DOMExceptionDataError", + ImportKeyError::Spki(_) => "DOMExceptionDataError", + ImportKeyError::InvalidP256ECSPKIData => "DOMExceptionDataError", + ImportKeyError::InvalidP384ECSPKIData => "DOMExceptionDataError", + ImportKeyError::InvalidP521ECSPKIData => "DOMExceptionDataError", + ImportKeyError::Der(_) => "DOMExceptionDataError", + } +} + +fn get_crypto_x448_error_class(e: &deno_crypto::X448Error) -> &'static str { + match e { + deno_crypto::X448Error::FailedExport => "DOMExceptionOperationError", + deno_crypto::X448Error::Der(_) => "Error", + } +} + +fn get_crypto_x25519_error_class(e: &deno_crypto::X25519Error) -> &'static str { + match e { + deno_crypto::X25519Error::FailedExport => "DOMExceptionOperationError", + deno_crypto::X25519Error::Der(_) => "Error", + } +} + +fn get_crypto_error_class(e: &deno_crypto::Error) -> &'static str { + match e { + deno_crypto::Error::Der(_) => "Error", + deno_crypto::Error::JoinError(_) => "Error", + deno_crypto::Error::MissingArgumentHash => "TypeError", + deno_crypto::Error::MissingArgumentSaltLength => "TypeError", + deno_crypto::Error::Other(e) => get_error_class_name(e).unwrap_or("Error"), + deno_crypto::Error::UnsupportedAlgorithm => "TypeError", + deno_crypto::Error::KeyRejected(_) => "Error", + deno_crypto::Error::RSA(_) => "Error", + deno_crypto::Error::Pkcs1(_) => "Error", + deno_crypto::Error::Unspecified(_) => "Error", + deno_crypto::Error::InvalidKeyFormat => "TypeError", + deno_crypto::Error::MissingArgumentPublicKey => "TypeError", + deno_crypto::Error::P256Ecdsa(_) => "Error", + deno_crypto::Error::DecodePrivateKey => "TypeError", + deno_crypto::Error::MissingArgumentNamedCurve => "TypeError", + deno_crypto::Error::MissingArgumentInfo => "TypeError", + deno_crypto::Error::HKDFLengthTooLarge => "DOMExceptionOperationError", + deno_crypto::Error::General(e) => get_crypto_shared_error_class(e), + deno_crypto::Error::Base64Decode(_) => "Error", + deno_crypto::Error::DataInvalidSize => "TypeError", + deno_crypto::Error::InvalidKeyLength => "TypeError", + deno_crypto::Error::EncryptionError => "DOMExceptionOperationError", + deno_crypto::Error::DecryptionError => "DOMExceptionOperationError", + deno_crypto::Error::ArrayBufferViewLengthExceeded(_) => { + "DOMExceptionQuotaExceededError" + } + } +} + +fn get_napi_error_class(e: &NApiError) -> &'static str { + match e { + NApiError::InvalidPath + | NApiError::LibLoading(_) + | NApiError::ModuleNotFound(_) => "TypeError", + NApiError::Permission(e) => get_permission_check_error_class(e), + } +} + +fn get_web_error_class(e: &WebError) -> &'static str { + match e { + WebError::Base64Decode => "DOMExceptionInvalidCharacterError", + WebError::InvalidEncodingLabel(_) => "RangeError", + WebError::BufferTooLong => "TypeError", + WebError::ValueTooLarge => "RangeError", + WebError::BufferTooSmall => "RangeError", + WebError::DataInvalid => "TypeError", + WebError::DataError(_) => "Error", + } +} + +fn get_web_compression_error_class(e: &CompressionError) -> &'static str { + match e { + CompressionError::UnsupportedFormat => "TypeError", + CompressionError::ResourceClosed => "TypeError", + CompressionError::IoTypeError(_) => "TypeError", + CompressionError::Io(e) => get_io_error_class(e), + } +} + +fn get_web_message_port_error_class(e: &MessagePortError) -> &'static str { + match e { + MessagePortError::InvalidTransfer => "TypeError", + MessagePortError::NotReady => "TypeError", + MessagePortError::TransferSelf => "TypeError", + MessagePortError::Canceled(e) => { + let io_err: io::Error = e.to_owned().into(); + get_io_error_class(&io_err) + } + MessagePortError::Resource(e) => get_error_class_name(e).unwrap_or("Error"), + } +} + +fn get_web_stream_resource_error_class( + e: &StreamResourceError, +) -> &'static str { + match e { + StreamResourceError::Canceled(e) => { + let io_err: io::Error = e.to_owned().into(); + get_io_error_class(&io_err) + } + StreamResourceError::Js(_) => "TypeError", + } +} + +fn get_web_blob_error_class(e: &BlobError) -> &'static str { + match e { + BlobError::BlobPartNotFound => "TypeError", + BlobError::SizeLargerThanBlobPart => "TypeError", + BlobError::BlobURLsNotSupported => "TypeError", + BlobError::Url(_) => "Error", + } +} + +fn get_ffi_repr_error_class(e: &ReprError) -> &'static str { + match e { + ReprError::InvalidOffset => "TypeError", + ReprError::InvalidArrayBuffer => "TypeError", + ReprError::DestinationLengthTooShort => "RangeError", + ReprError::InvalidCString => "TypeError", + ReprError::CStringTooLong => "TypeError", + ReprError::InvalidBool => "TypeError", + ReprError::InvalidU8 => "TypeError", + ReprError::InvalidI8 => "TypeError", + ReprError::InvalidU16 => "TypeError", + ReprError::InvalidI16 => "TypeError", + ReprError::InvalidU32 => "TypeError", + ReprError::InvalidI32 => "TypeError", + ReprError::InvalidU64 => "TypeError", + ReprError::InvalidI64 => "TypeError", + ReprError::InvalidF32 => "TypeError", + ReprError::InvalidF64 => "TypeError", + ReprError::InvalidPointer => "TypeError", + ReprError::Permission(e) => get_permission_check_error_class(e), + } +} + +fn get_ffi_dlfcn_error_class(e: &DlfcnError) -> &'static str { + match e { + DlfcnError::RegisterSymbol { .. } => "Error", + DlfcnError::Dlopen(_) => "Error", + DlfcnError::Permission(e) => get_permission_check_error_class(e), + DlfcnError::Other(e) => get_error_class_name(e).unwrap_or("Error"), + } +} + +fn get_ffi_static_error_class(e: &StaticError) -> &'static str { + match e { + StaticError::Dlfcn(e) => get_ffi_dlfcn_error_class(e), + StaticError::InvalidTypeVoid => "TypeError", + StaticError::InvalidTypeStruct => "TypeError", + StaticError::Resource(e) => get_error_class_name(e).unwrap_or("Error"), + } +} + +fn get_ffi_callback_error_class(e: &CallbackError) -> &'static str { + match e { + CallbackError::Resource(e) => get_error_class_name(e).unwrap_or("Error"), + CallbackError::Other(e) => get_error_class_name(e).unwrap_or("Error"), + CallbackError::Permission(e) => get_permission_check_error_class(e), + } +} + +fn get_ffi_call_error_class(e: &CallError) -> &'static str { + match e { + CallError::IR(_) => "TypeError", + CallError::NonblockingCallFailure(_) => "Error", + CallError::InvalidSymbol(_) => "TypeError", + CallError::Permission(e) => get_permission_check_error_class(e), + CallError::Callback(e) => get_ffi_callback_error_class(e), + CallError::Resource(e) => get_error_class_name(e).unwrap_or("Error"), + } +} + +fn get_webstorage_class_name(e: &WebStorageError) -> &'static str { + match e { + WebStorageError::ContextNotSupported => "DOMExceptionNotSupportedError", + WebStorageError::Sqlite(_) => "Error", + WebStorageError::Io(e) => get_io_error_class(e), + WebStorageError::StorageExceeded => "DOMExceptionQuotaExceededError", + } +} + +fn get_tls_error_class(e: &TlsError) -> &'static str { + match e { + TlsError::Rustls(_) => "Error", + TlsError::UnableAddPemFileToCert(e) => get_io_error_class(e), + TlsError::CertInvalid + | TlsError::CertsNotFound + | TlsError::KeysNotFound + | TlsError::KeyDecode => "InvalidData", + } +} + +pub fn get_cron_error_class(e: &CronError) -> &'static str { + match e { + CronError::Resource(e) => { + deno_core::error::get_custom_error_class(e).unwrap_or("Error") + } + CronError::NameExceeded(_) => "TypeError", + CronError::NameInvalid => "TypeError", + CronError::AlreadyExists => "TypeError", + CronError::TooManyCrons => "TypeError", + CronError::InvalidCron => "TypeError", + CronError::InvalidBackoff => "TypeError", + CronError::AcquireError(_) => "Error", + CronError::Other(e) => get_error_class_name(e).unwrap_or("Error"), + } +} + +fn get_canvas_error(e: &CanvasError) -> &'static str { + match e { + CanvasError::UnsupportedColorType(_) => "TypeError", + CanvasError::Image(_) => "Error", + } +} + +pub fn get_cache_error(error: &CacheError) -> &'static str { + match error { + CacheError::Sqlite(_) => "Error", + CacheError::JoinError(_) => "Error", + CacheError::Resource(err) => { + deno_core::error::get_custom_error_class(err).unwrap_or("Error") + } + CacheError::Other(e) => get_error_class_name(e).unwrap_or("Error"), + CacheError::Io(err) => get_io_error_class(err), + } +} + +fn get_broadcast_channel_error(error: &BroadcastChannelError) -> &'static str { + match error { + BroadcastChannelError::Resource(err) => { + deno_core::error::get_custom_error_class(err).unwrap() + } + BroadcastChannelError::MPSCSendError(_) => "Error", + BroadcastChannelError::BroadcastSendError(_) => "Error", + BroadcastChannelError::Other(err) => { + get_error_class_name(err).unwrap_or("Error") + } + } +} + +fn get_fetch_error(error: &FetchError) -> &'static str { + match error { + FetchError::Resource(e) => get_error_class_name(e).unwrap_or("Error"), + FetchError::Permission(e) => get_permission_check_error_class(e), + FetchError::NetworkError => "TypeError", + FetchError::FsNotGet(_) => "TypeError", + FetchError::InvalidUrl(_) => "TypeError", + FetchError::InvalidHeaderName(_) => "TypeError", + FetchError::InvalidHeaderValue(_) => "TypeError", + FetchError::DataUrl(_) => "TypeError", + FetchError::Base64(_) => "TypeError", + FetchError::BlobNotFound => "TypeError", + FetchError::SchemeNotSupported(_) => "TypeError", + FetchError::RequestCanceled => "TypeError", + FetchError::Http(_) => "Error", + FetchError::ClientCreate(e) => get_http_client_create_error(e), + FetchError::Url(e) => get_url_parse_error_class(e), + FetchError::Method(_) => "TypeError", + FetchError::ClientSend(_) => "TypeError", + FetchError::RequestBuilderHook(_) => "TypeError", + FetchError::Io(e) => get_io_error_class(e), + FetchError::Hyper(e) => get_hyper_error_class(e), + } +} + +fn get_http_client_create_error(error: &HttpClientCreateError) -> &'static str { + match error { + HttpClientCreateError::Tls(_) => "TypeError", + HttpClientCreateError::InvalidUserAgent(_) => "TypeError", + HttpClientCreateError::InvalidProxyUrl => "TypeError", + HttpClientCreateError::HttpVersionSelectionInvalid => "TypeError", + HttpClientCreateError::RootCertStore(_) => "TypeError", + } +} + +fn get_websocket_error(error: &WebsocketError) -> &'static str { + match error { + WebsocketError::Resource(e) => get_error_class_name(e).unwrap_or("Error"), + WebsocketError::Permission(e) => get_permission_check_error_class(e), + WebsocketError::Url(e) => get_url_parse_error_class(e), + WebsocketError::Io(e) => get_io_error_class(e), + WebsocketError::WebSocket(_) => "TypeError", + WebsocketError::ConnectionFailed(_) => "DOMExceptionNetworkError", + WebsocketError::Uri(_) => "Error", + WebsocketError::Canceled(e) => { + let io_err: io::Error = e.to_owned().into(); + get_io_error_class(&io_err) + } + } +} + +fn get_websocket_handshake_error(error: &HandshakeError) -> &'static str { + match error { + HandshakeError::RootStoreError(e) => { + get_error_class_name(e).unwrap_or("Error") + } + HandshakeError::Tls(e) => get_tls_error_class(e), + HandshakeError::MissingPath => "TypeError", + HandshakeError::Http(_) => "Error", + HandshakeError::InvalidHostname(_) => "TypeError", + HandshakeError::Io(e) => get_io_error_class(e), + HandshakeError::Rustls(_) => "Error", + HandshakeError::H2(_) => "Error", + HandshakeError::NoH2Alpn => "Error", + HandshakeError::InvalidStatusCode(_) => "Error", + HandshakeError::WebSocket(_) => "TypeError", + HandshakeError::HeaderName(_) => "TypeError", + HandshakeError::HeaderValue(_) => "TypeError", + } +} + +fn get_fs_ops_error(error: &FsOpsError) -> &'static str { + use FsOpsErrorKind::*; + match error.as_kind() { + Io(e) => get_io_error_class(e), + OperationError(e) => get_fs_error(&e.err), + Permission(e) => get_permission_check_error_class(e), + Resource(e) | Other(e) => get_error_class_name(e).unwrap_or("Error"), + InvalidUtf8(_) => "InvalidData", + StripPrefix(_) => "Error", + Canceled(e) => { + let io_err: io::Error = e.to_owned().into(); + get_io_error_class(&io_err) + } + InvalidSeekMode(_) => "TypeError", + InvalidControlCharacter(_) => "Error", + InvalidCharacter(_) => "Error", + #[cfg(windows)] + InvalidTrailingCharacter => "Error", + NotCapableAccess { .. } => "NotCapable", + NotCapable(_) => "NotCapable", + } +} + +fn get_kv_error(error: &KvError) -> &'static str { + use KvErrorKind::*; + match error.as_kind() { + DatabaseHandler(e) | Resource(e) | Kv(e) => { + get_error_class_name(e).unwrap_or("Error") + } + TooManyRanges(_) => "TypeError", + TooManyEntries(_) => "TypeError", + TooManyChecks(_) => "TypeError", + TooManyMutations(_) => "TypeError", + TooManyKeys(_) => "TypeError", + InvalidLimit => "TypeError", + InvalidBoundaryKey => "TypeError", + KeyTooLargeToRead(_) => "TypeError", + KeyTooLargeToWrite(_) => "TypeError", + TotalMutationTooLarge(_) => "TypeError", + TotalKeyTooLarge(_) => "TypeError", + Io(e) => get_io_error_class(e), + QueueMessageNotFound => "TypeError", + StartKeyNotInKeyspace => "TypeError", + EndKeyNotInKeyspace => "TypeError", + StartKeyGreaterThanEndKey => "TypeError", + InvalidCheck(e) => match e { + KvCheckError::InvalidVersionstamp => "TypeError", + KvCheckError::Io(e) => get_io_error_class(e), + }, + InvalidMutation(e) => match e { + KvMutationError::BigInt(_) => "Error", + KvMutationError::Io(e) => get_io_error_class(e), + KvMutationError::InvalidMutationWithValue(_) => "TypeError", + KvMutationError::InvalidMutationWithoutValue(_) => "TypeError", + }, + InvalidEnqueue(e) => get_io_error_class(e), + EmptyKey => "TypeError", + ValueTooLarge(_) => "TypeError", + EnqueuePayloadTooLarge(_) => "TypeError", + InvalidCursor => "TypeError", + CursorOutOfBounds => "TypeError", + InvalidRange => "TypeError", + } +} + +fn get_net_error(error: &NetError) -> &'static str { + match error { + NetError::ListenerClosed => "BadResource", + NetError::ListenerBusy => "Busy", + NetError::SocketClosed => "BadResource", + NetError::SocketClosedNotConnected => "NotConnected", + NetError::SocketBusy => "Busy", + NetError::Io(e) => get_io_error_class(e), + NetError::AcceptTaskOngoing => "Busy", + NetError::RootCertStore(e) | NetError::Resource(e) => { + get_error_class_name(e).unwrap_or("Error") + } + NetError::Permission(e) => get_permission_check_error_class(e), + NetError::NoResolvedAddress => "Error", + NetError::AddrParse(_) => "Error", + NetError::Map(e) => get_net_map_error(e), + NetError::Canceled(e) => { + let io_err: io::Error = e.to_owned().into(); + get_io_error_class(&io_err) + } + NetError::DnsNotFound(_) => "NotFound", + NetError::DnsNotConnected(_) => "NotConnected", + NetError::DnsTimedOut(_) => "TimedOut", + NetError::Dns(_) => "Error", + NetError::UnsupportedRecordType => "NotSupported", + NetError::InvalidUtf8(_) => "InvalidData", + NetError::UnexpectedKeyType => "Error", + NetError::InvalidHostname(_) => "TypeError", + NetError::TcpStreamBusy => "Busy", + NetError::Rustls(_) => "Error", + NetError::Tls(e) => get_tls_error_class(e), + NetError::ListenTlsRequiresKey => "InvalidData", + NetError::Reunite(_) => "Error", + } +} + +fn get_net_map_error(error: &deno_net::io::MapError) -> &'static str { + match error { + deno_net::io::MapError::Io(e) => get_io_error_class(e), + deno_net::io::MapError::NoResources => "Error", + } +} + +fn get_child_permission_error(e: &ChildPermissionError) -> &'static str { + match e { + ChildPermissionError::Escalation => "NotCapable", + ChildPermissionError::PathResolve(e) => get_path_resolve_error(e), + ChildPermissionError::NetDescriptorParse(_) => "URIError", + ChildPermissionError::EnvDescriptorParse(_) => "Error", + ChildPermissionError::SysDescriptorParse(e) => { + get_sys_descriptor_parse_error(e) + } + ChildPermissionError::RunDescriptorParse(e) => { + get_run_descriptor_parse_error(e) + } + } +} + +fn get_create_worker_error(error: &CreateWorkerError) -> &'static str { + match error { + CreateWorkerError::ClassicWorkers => "DOMExceptionNotSupportedError", + CreateWorkerError::Permission(e) => get_child_permission_error(e), + CreateWorkerError::ModuleResolution(e) => { + get_module_resolution_error_class(e) + } + CreateWorkerError::Io(e) => get_io_error_class(e), + CreateWorkerError::MessagePort(e) => get_web_message_port_error_class(e), + } +} + +fn get_tty_error(error: &TtyError) -> &'static str { + match error { + TtyError::Resource(e) | TtyError::Other(e) => { + get_error_class_name(e).unwrap_or("Error") + } + TtyError::Io(e) => get_io_error_class(e), + #[cfg(unix)] + TtyError::Nix(e) => get_nix_error_class(e), + } +} + +fn get_readline_error(error: &ReadlineError) -> &'static str { + match error { + ReadlineError::Io(e) => get_io_error_class(e), + ReadlineError::Eof => "Error", + ReadlineError::Interrupted => "Error", + #[cfg(unix)] + ReadlineError::Errno(e) => get_nix_error_class(e), + ReadlineError::WindowResized => "Error", + #[cfg(windows)] + ReadlineError::Decode(_) => "Error", + #[cfg(windows)] + ReadlineError::SystemError(_) => "Error", + _ => "Error", + } +} + +fn get_signal_error(error: &SignalError) -> &'static str { + match error { + SignalError::InvalidSignalStr(_) => "TypeError", + SignalError::InvalidSignalInt(_) => "TypeError", + SignalError::SignalNotAllowed(_) => "TypeError", + SignalError::Io(e) => get_io_error_class(e), + } +} + +fn get_fs_events_error(error: &FsEventsError) -> &'static str { + match error { + FsEventsError::Resource(e) => get_error_class_name(e).unwrap_or("Error"), + FsEventsError::Permission(e) => get_permission_check_error_class(e), + FsEventsError::Notify(e) => get_notify_error_class(e), + FsEventsError::Canceled(e) => { + let io_err: io::Error = e.to_owned().into(); + get_io_error_class(&io_err) + } + } +} + +fn get_http_start_error(error: &HttpStartError) -> &'static str { + match error { + HttpStartError::TcpStreamInUse => "Busy", + HttpStartError::TlsStreamInUse => "Busy", + HttpStartError::UnixSocketInUse => "Busy", + HttpStartError::ReuniteTcp(_) => "Error", + #[cfg(unix)] + HttpStartError::ReuniteUnix(_) => "Error", + HttpStartError::Io(e) => get_io_error_class(e), + HttpStartError::Other(e) => get_error_class_name(e).unwrap_or("Error"), + } +} + +fn get_process_error(error: &ProcessError) -> &'static str { + match error { + ProcessError::SpawnFailed { error, .. } => get_process_error(error), + ProcessError::FailedResolvingCwd(e) | ProcessError::Io(e) => { + get_io_error_class(e) + } + ProcessError::Permission(e) => get_permission_check_error_class(e), + ProcessError::Resource(e) => get_error_class_name(e).unwrap_or("Error"), + ProcessError::BorrowMut(_) => "Error", + ProcessError::Which(_) => "Error", + ProcessError::ChildProcessAlreadyTerminated => "TypeError", + ProcessError::Signal(e) => get_signal_error(e), + ProcessError::MissingCmd => "Error", + ProcessError::InvalidPid => "TypeError", + #[cfg(unix)] + ProcessError::Nix(e) => get_nix_error_class(e), + ProcessError::RunPermission(e) => match e { + CheckRunPermissionError::Permission(e) => { + get_permission_check_error_class(e) + } + CheckRunPermissionError::Other(e) => { + get_error_class_name(e).unwrap_or("Error") + } + }, + } +} + +fn get_http_error(error: &HttpError) -> &'static str { + match error { + HttpError::Canceled(e) => { + let io_err: io::Error = e.to_owned().into(); + get_io_error_class(&io_err) + } + HttpError::HyperV014(e) => get_hyper_v014_error_class(e), + HttpError::InvalidHeaderName(_) => "Error", + HttpError::InvalidHeaderValue(_) => "Error", + HttpError::Http(_) => "Error", + HttpError::ResponseHeadersAlreadySent => "Http", + HttpError::ConnectionClosedWhileSendingResponse => "Http", + HttpError::AlreadyInUse => "Http", + HttpError::Io(e) => get_io_error_class(e), + HttpError::NoResponseHeaders => "Http", + HttpError::ResponseAlreadyCompleted => "Http", + HttpError::UpgradeBodyUsed => "Http", + HttpError::Resource(e) | HttpError::Other(e) => { + get_error_class_name(e).unwrap_or("Error") + } + } +} + +fn get_http_next_error(error: &HttpNextError) -> &'static str { + match error { + HttpNextError::Io(e) => get_io_error_class(e), + HttpNextError::WebSocketUpgrade(e) => get_websocket_upgrade_error(e), + HttpNextError::Hyper(e) => get_hyper_error_class(e), + HttpNextError::JoinError(_) => "Error", + HttpNextError::Canceled(e) => { + let io_err: io::Error = e.to_owned().into(); + get_io_error_class(&io_err) + } + HttpNextError::UpgradeUnavailable(_) => "Error", + HttpNextError::HttpPropertyExtractor(e) | HttpNextError::Resource(e) => { + get_error_class_name(e).unwrap_or("Error") + } + } +} + +fn get_websocket_upgrade_error(error: &WebSocketUpgradeError) -> &'static str { + match error { + WebSocketUpgradeError::InvalidHeaders => "Http", + WebSocketUpgradeError::HttpParse(_) => "Error", + WebSocketUpgradeError::Http(_) => "Error", + WebSocketUpgradeError::Utf8(_) => "Error", + WebSocketUpgradeError::InvalidHeaderName(_) => "Error", + WebSocketUpgradeError::InvalidHeaderValue(_) => "Error", + WebSocketUpgradeError::InvalidHttpStatusLine => "Http", + WebSocketUpgradeError::UpgradeBufferAlreadyCompleted => "Http", + } +} + +fn get_fs_error(e: &FsError) -> &'static str { + match &e { + FsError::Io(e) => get_io_error_class(e), + FsError::FileBusy => "Busy", + FsError::NotSupported => "NotSupported", + FsError::NotCapable(_) => "NotCapable", + } +} + +mod node { + use super::get_error_class_name; + use super::get_io_error_class; + use super::get_permission_check_error_class; + use super::get_serde_json_error_class; + use super::get_url_parse_error_class; + pub use deno_node::ops::blocklist::BlocklistError; + pub use deno_node::ops::crypto::cipher::CipherContextError; + pub use deno_node::ops::crypto::cipher::CipherError; + pub use deno_node::ops::crypto::cipher::DecipherContextError; + pub use deno_node::ops::crypto::cipher::DecipherError; + pub use deno_node::ops::crypto::digest::HashError; + pub use deno_node::ops::crypto::keys::AsymmetricPrivateKeyDerError; + pub use deno_node::ops::crypto::keys::AsymmetricPrivateKeyError; + pub use deno_node::ops::crypto::keys::AsymmetricPublicKeyDerError; + pub use deno_node::ops::crypto::keys::AsymmetricPublicKeyError; + pub use deno_node::ops::crypto::keys::AsymmetricPublicKeyJwkError; + pub use deno_node::ops::crypto::keys::EcJwkError; + pub use deno_node::ops::crypto::keys::EdRawError; + pub use deno_node::ops::crypto::keys::ExportPrivateKeyPemError; + pub use deno_node::ops::crypto::keys::ExportPublicKeyPemError; + pub use deno_node::ops::crypto::keys::GenerateRsaPssError; + pub use deno_node::ops::crypto::keys::RsaJwkError; + pub use deno_node::ops::crypto::keys::RsaPssParamsParseError; + pub use deno_node::ops::crypto::keys::X509PublicKeyError; + pub use deno_node::ops::crypto::sign::KeyObjectHandlePrehashedSignAndVerifyError; + pub use deno_node::ops::crypto::x509::X509Error; + pub use deno_node::ops::crypto::DiffieHellmanError; + pub use deno_node::ops::crypto::EcdhEncodePubKey; + pub use deno_node::ops::crypto::HkdfError; + pub use deno_node::ops::crypto::Pbkdf2Error; + pub use deno_node::ops::crypto::PrivateEncryptDecryptError; + pub use deno_node::ops::crypto::ScryptAsyncError; + pub use deno_node::ops::crypto::SignEd25519Error; + pub use deno_node::ops::crypto::VerifyEd25519Error; + pub use deno_node::ops::fs::FsError; + pub use deno_node::ops::http2::Http2Error; + pub use deno_node::ops::idna::IdnaError; + pub use deno_node::ops::ipc::IpcError; + pub use deno_node::ops::ipc::IpcJsonStreamError; + use deno_node::ops::os::priority::PriorityError; + pub use deno_node::ops::os::OsError; + pub use deno_node::ops::require::RequireError; + use deno_node::ops::require::RequireErrorKind; + pub use deno_node::ops::worker_threads::WorkerThreadsFilenameError; + pub use deno_node::ops::zlib::brotli::BrotliError; + pub use deno_node::ops::zlib::mode::ModeError; + pub use deno_node::ops::zlib::ZlibError; + + pub fn get_blocklist_error(error: &BlocklistError) -> &'static str { + match error { + BlocklistError::AddrParse(_) => "Error", + BlocklistError::IpNetwork(_) => "Error", + BlocklistError::InvalidAddress => "Error", + BlocklistError::IpVersionMismatch => "Error", + } + } + + pub fn get_fs_error(error: &FsError) -> &'static str { + match error { + FsError::Permission(e) => get_permission_check_error_class(e), + FsError::Io(e) => get_io_error_class(e), + #[cfg(windows)] + FsError::PathHasNoRoot => "Error", + #[cfg(not(any(unix, windows)))] + FsError::UnsupportedPlatform => "Error", + FsError::Fs(e) => super::get_fs_error(e), + } + } + + pub fn get_idna_error(error: &IdnaError) -> &'static str { + match error { + IdnaError::InvalidInput => "RangeError", + IdnaError::InputTooLong => "Error", + IdnaError::IllegalInput => "RangeError", + } + } + + pub fn get_ipc_json_stream_error(error: &IpcJsonStreamError) -> &'static str { + match error { + IpcJsonStreamError::Io(e) => get_io_error_class(e), + IpcJsonStreamError::SimdJson(_) => "Error", + } + } + + pub fn get_ipc_error(error: &IpcError) -> &'static str { + match error { + IpcError::Resource(e) => get_error_class_name(e).unwrap_or("Error"), + IpcError::IpcJsonStream(e) => get_ipc_json_stream_error(e), + IpcError::Canceled(e) => { + let io_err: std::io::Error = e.to_owned().into(); + get_io_error_class(&io_err) + } + IpcError::SerdeJson(e) => get_serde_json_error_class(e), + } + } + + pub fn get_worker_threads_filename_error( + error: &WorkerThreadsFilenameError, + ) -> &'static str { + match error { + WorkerThreadsFilenameError::Permission(e) => { + get_error_class_name(e).unwrap_or("Error") + } + WorkerThreadsFilenameError::UrlParse(e) => get_url_parse_error_class(e), + WorkerThreadsFilenameError::InvalidRelativeUrl => "Error", + WorkerThreadsFilenameError::UrlFromPathString => "Error", + WorkerThreadsFilenameError::UrlToPathString => "Error", + WorkerThreadsFilenameError::UrlToPath => "Error", + WorkerThreadsFilenameError::FileNotFound(_) => "Error", + WorkerThreadsFilenameError::Fs(e) => super::get_fs_error(e), + } + } + + pub fn get_require_error(error: &RequireError) -> &'static str { + use RequireErrorKind::*; + match error.as_kind() { + UrlParse(e) => get_url_parse_error_class(e), + Permission(e) => get_error_class_name(e).unwrap_or("Error"), + PackageExportsResolve(_) + | PackageJsonLoad(_) + | ClosestPkgJson(_) + | FilePathConversion(_) + | UrlConversion(_) + | ReadModule(_) + | PackageImportsResolve(_) => "Error", + Fs(e) | UnableToGetCwd(e) => super::get_fs_error(e), + } + } + + pub fn get_http2_error(error: &Http2Error) -> &'static str { + match error { + Http2Error::Resource(e) => get_error_class_name(e).unwrap_or("Error"), + Http2Error::UrlParse(e) => get_url_parse_error_class(e), + Http2Error::H2(_) => "Error", + } + } + + pub fn get_os_error(error: &OsError) -> &'static str { + match error { + OsError::Priority(e) => match e { + PriorityError::Io(e) => get_io_error_class(e), + #[cfg(windows)] + PriorityError::InvalidPriority => "TypeError", + }, + OsError::Permission(e) => get_permission_check_error_class(e), + OsError::FailedToGetCpuInfo => "TypeError", + OsError::FailedToGetUserInfo(e) => get_io_error_class(e), + } + } + + pub fn get_brotli_error(error: &BrotliError) -> &'static str { + match error { + BrotliError::InvalidEncoderMode => "TypeError", + BrotliError::CompressFailed => "TypeError", + BrotliError::DecompressFailed => "TypeError", + BrotliError::Join(_) => "Error", + BrotliError::Resource(e) => get_error_class_name(e).unwrap_or("Error"), + BrotliError::Io(e) => get_io_error_class(e), + } + } + + pub fn get_mode_error(_: &ModeError) -> &'static str { + "Error" + } + + pub fn get_zlib_error(e: &ZlibError) -> &'static str { + match e { + ZlibError::NotInitialized => "TypeError", + ZlibError::Mode(e) => get_mode_error(e), + ZlibError::Other(e) => get_error_class_name(e).unwrap_or("Error"), + } + } + + pub fn get_crypto_cipher_context_error( + e: &CipherContextError, + ) -> &'static str { + match e { + CipherContextError::ContextInUse => "TypeError", + CipherContextError::Cipher(e) => get_crypto_cipher_error(e), + CipherContextError::Resource(e) => { + get_error_class_name(e).unwrap_or("Error") + } + } + } + + pub fn get_crypto_cipher_error(e: &CipherError) -> &'static str { + match e { + CipherError::InvalidIvLength => "TypeError", + CipherError::InvalidKeyLength => "RangeError", + CipherError::InvalidInitializationVector => "TypeError", + CipherError::CannotPadInputData => "TypeError", + CipherError::UnknownCipher(_) => "TypeError", + } + } + + pub fn get_crypto_decipher_context_error( + e: &DecipherContextError, + ) -> &'static str { + match e { + DecipherContextError::ContextInUse => "TypeError", + DecipherContextError::Decipher(e) => get_crypto_decipher_error(e), + DecipherContextError::Resource(e) => { + get_error_class_name(e).unwrap_or("Error") + } + } + } + + pub fn get_crypto_decipher_error(e: &DecipherError) -> &'static str { + match e { + DecipherError::InvalidIvLength => "TypeError", + DecipherError::InvalidKeyLength => "RangeError", + DecipherError::InvalidInitializationVector => "TypeError", + DecipherError::CannotUnpadInputData => "TypeError", + DecipherError::DataAuthenticationFailed => "TypeError", + DecipherError::SetAutoPaddingFalseAes128GcmUnsupported => "TypeError", + DecipherError::SetAutoPaddingFalseAes256GcmUnsupported => "TypeError", + DecipherError::UnknownCipher(_) => "TypeError", + } + } + + pub fn get_x509_error(_: &X509Error) -> &'static str { + "Error" + } + + pub fn get_crypto_key_object_handle_prehashed_sign_and_verify_error( + e: &KeyObjectHandlePrehashedSignAndVerifyError, + ) -> &'static str { + match e { + KeyObjectHandlePrehashedSignAndVerifyError::InvalidDsaSignatureEncoding => "TypeError", + KeyObjectHandlePrehashedSignAndVerifyError::KeyIsNotPrivate => "TypeError", + KeyObjectHandlePrehashedSignAndVerifyError::DigestNotAllowedForRsaSignature(_) => "TypeError", + KeyObjectHandlePrehashedSignAndVerifyError::FailedToSignDigestWithRsa => "Error", + KeyObjectHandlePrehashedSignAndVerifyError::DigestNotAllowedForRsaPssSignature(_) => "TypeError", + KeyObjectHandlePrehashedSignAndVerifyError::FailedToSignDigestWithRsaPss => "Error", + KeyObjectHandlePrehashedSignAndVerifyError::FailedToSignDigestWithDsa => "TypeError", + KeyObjectHandlePrehashedSignAndVerifyError::RsaPssHashAlgorithmUnsupported => "TypeError", + KeyObjectHandlePrehashedSignAndVerifyError::PrivateKeyDisallowsUsage { .. } => "TypeError", + KeyObjectHandlePrehashedSignAndVerifyError::FailedToSignDigest => "TypeError", + KeyObjectHandlePrehashedSignAndVerifyError::X25519KeyCannotBeUsedForSigning => "TypeError", + KeyObjectHandlePrehashedSignAndVerifyError::Ed25519KeyCannotBeUsedForPrehashedSigning => "TypeError", + KeyObjectHandlePrehashedSignAndVerifyError::DhKeyCannotBeUsedForSigning => "TypeError", + KeyObjectHandlePrehashedSignAndVerifyError::KeyIsNotPublicOrPrivate => "TypeError", + KeyObjectHandlePrehashedSignAndVerifyError::InvalidDsaSignature => "TypeError", + KeyObjectHandlePrehashedSignAndVerifyError::X25519KeyCannotBeUsedForVerification => "TypeError", + KeyObjectHandlePrehashedSignAndVerifyError::Ed25519KeyCannotBeUsedForPrehashedVerification => "TypeError", + KeyObjectHandlePrehashedSignAndVerifyError::DhKeyCannotBeUsedForVerification => "TypeError", + } + } + + pub fn get_crypto_hash_error(_: &HashError) -> &'static str { + "Error" + } + + pub fn get_asymmetric_public_key_jwk_error( + e: &AsymmetricPublicKeyJwkError, + ) -> &'static str { + match e { + AsymmetricPublicKeyJwkError::UnsupportedJwkEcCurveP224 => "TypeError", + AsymmetricPublicKeyJwkError::JwkExportNotImplementedForKeyType => { + "TypeError" + } + AsymmetricPublicKeyJwkError::KeyIsNotAsymmetricPublicKey => "TypeError", + } + } + + pub fn get_generate_rsa_pss_error(_: &GenerateRsaPssError) -> &'static str { + "TypeError" + } + + pub fn get_asymmetric_private_key_der_error( + e: &AsymmetricPrivateKeyDerError, + ) -> &'static str { + match e { + AsymmetricPrivateKeyDerError::KeyIsNotAsymmetricPrivateKey => "TypeError", + AsymmetricPrivateKeyDerError::InvalidRsaPrivateKey => "TypeError", + AsymmetricPrivateKeyDerError::ExportingNonRsaPrivateKeyAsPkcs1Unsupported => "TypeError", + AsymmetricPrivateKeyDerError::InvalidEcPrivateKey => "TypeError", + AsymmetricPrivateKeyDerError::ExportingNonEcPrivateKeyAsSec1Unsupported => "TypeError", + AsymmetricPrivateKeyDerError::ExportingNonRsaPssPrivateKeyAsPkcs8Unsupported => "Error", + AsymmetricPrivateKeyDerError::InvalidDsaPrivateKey => "TypeError", + AsymmetricPrivateKeyDerError::InvalidX25519PrivateKey => "TypeError", + AsymmetricPrivateKeyDerError::InvalidEd25519PrivateKey => "TypeError", + AsymmetricPrivateKeyDerError::InvalidDhPrivateKey => "TypeError", + AsymmetricPrivateKeyDerError::UnsupportedKeyType(_) => "TypeError", + } + } + + pub fn get_asymmetric_public_key_der_error( + _: &AsymmetricPublicKeyDerError, + ) -> &'static str { + "TypeError" + } + + pub fn get_export_public_key_pem_error( + e: &ExportPublicKeyPemError, + ) -> &'static str { + match e { + ExportPublicKeyPemError::AsymmetricPublicKeyDer(e) => { + get_asymmetric_public_key_der_error(e) + } + ExportPublicKeyPemError::VeryLargeData => "TypeError", + ExportPublicKeyPemError::Der(_) => "Error", + } + } + + pub fn get_export_private_key_pem_error( + e: &ExportPrivateKeyPemError, + ) -> &'static str { + match e { + ExportPrivateKeyPemError::AsymmetricPublicKeyDer(e) => { + get_asymmetric_private_key_der_error(e) + } + ExportPrivateKeyPemError::VeryLargeData => "TypeError", + ExportPrivateKeyPemError::Der(_) => "Error", + } + } + + pub fn get_x509_public_key_error(e: &X509PublicKeyError) -> &'static str { + match e { + X509PublicKeyError::X509(_) => "Error", + X509PublicKeyError::Rsa(_) => "Error", + X509PublicKeyError::Asn1(_) => "Error", + X509PublicKeyError::Ec(_) => "Error", + X509PublicKeyError::UnsupportedEcNamedCurve => "TypeError", + X509PublicKeyError::MissingEcParameters => "TypeError", + X509PublicKeyError::MalformedDssPublicKey => "TypeError", + X509PublicKeyError::UnsupportedX509KeyType => "TypeError", + } + } + + pub fn get_rsa_jwk_error(e: &RsaJwkError) -> &'static str { + match e { + RsaJwkError::Base64(_) => "Error", + RsaJwkError::Rsa(_) => "Error", + RsaJwkError::MissingRsaPrivateComponent => "TypeError", + } + } + + pub fn get_ec_jwk_error(e: &EcJwkError) -> &'static str { + match e { + EcJwkError::Ec(_) => "Error", + EcJwkError::UnsupportedCurve(_) => "TypeError", + } + } + + pub fn get_ed_raw_error(e: &EdRawError) -> &'static str { + match e { + EdRawError::Ed25519Signature(_) => "Error", + EdRawError::InvalidEd25519Key => "TypeError", + EdRawError::UnsupportedCurve => "TypeError", + } + } + + pub fn get_pbkdf2_error(e: &Pbkdf2Error) -> &'static str { + match e { + Pbkdf2Error::UnsupportedDigest(_) => "TypeError", + Pbkdf2Error::Join(_) => "Error", + } + } + + pub fn get_scrypt_async_error(e: &ScryptAsyncError) -> &'static str { + match e { + ScryptAsyncError::Join(_) => "Error", + ScryptAsyncError::Other(e) => get_error_class_name(e).unwrap_or("Error"), + } + } + + pub fn get_hkdf_error_error(e: &HkdfError) -> &'static str { + match e { + HkdfError::ExpectedSecretKey => "TypeError", + HkdfError::HkdfExpandFailed => "TypeError", + HkdfError::UnsupportedDigest(_) => "TypeError", + HkdfError::Join(_) => "Error", + } + } + + pub fn get_rsa_pss_params_parse_error( + _: &RsaPssParamsParseError, + ) -> &'static str { + "TypeError" + } + + pub fn get_asymmetric_private_key_error( + e: &AsymmetricPrivateKeyError, + ) -> &'static str { + match e { + AsymmetricPrivateKeyError::InvalidPemPrivateKeyInvalidUtf8(_) => "TypeError", + AsymmetricPrivateKeyError::InvalidEncryptedPemPrivateKey => "TypeError", + AsymmetricPrivateKeyError::InvalidPemPrivateKey => "TypeError", + AsymmetricPrivateKeyError::EncryptedPrivateKeyRequiresPassphraseToDecrypt => "TypeError", + AsymmetricPrivateKeyError::InvalidPkcs1PrivateKey => "TypeError", + AsymmetricPrivateKeyError::InvalidSec1PrivateKey => "TypeError", + AsymmetricPrivateKeyError::UnsupportedPemLabel(_) => "TypeError", + AsymmetricPrivateKeyError::RsaPssParamsParse(e) => get_rsa_pss_params_parse_error(e), + AsymmetricPrivateKeyError::InvalidEncryptedPkcs8PrivateKey => "TypeError", + AsymmetricPrivateKeyError::InvalidPkcs8PrivateKey => "TypeError", + AsymmetricPrivateKeyError::Pkcs1PrivateKeyDoesNotSupportEncryptionWithPassphrase => "TypeError", + AsymmetricPrivateKeyError::Sec1PrivateKeyDoesNotSupportEncryptionWithPassphrase => "TypeError", + AsymmetricPrivateKeyError::UnsupportedEcNamedCurve => "TypeError", + AsymmetricPrivateKeyError::InvalidPrivateKey => "TypeError", + AsymmetricPrivateKeyError::InvalidDsaPrivateKey => "TypeError", + AsymmetricPrivateKeyError::MalformedOrMissingNamedCurveInEcParameters => "TypeError", + AsymmetricPrivateKeyError::UnsupportedKeyType(_) => "TypeError", + AsymmetricPrivateKeyError::UnsupportedKeyFormat(_) => "TypeError", + AsymmetricPrivateKeyError::InvalidX25519PrivateKey => "TypeError", + AsymmetricPrivateKeyError::X25519PrivateKeyIsWrongLength => "TypeError", + AsymmetricPrivateKeyError::InvalidEd25519PrivateKey => "TypeError", + AsymmetricPrivateKeyError::MissingDhParameters => "TypeError", + AsymmetricPrivateKeyError::UnsupportedPrivateKeyOid => "TypeError", + } + } + + pub fn get_asymmetric_public_key_error( + e: &AsymmetricPublicKeyError, + ) -> &'static str { + match e { + AsymmetricPublicKeyError::InvalidPemPrivateKeyInvalidUtf8(_) => { + "TypeError" + } + AsymmetricPublicKeyError::InvalidPemPublicKey => "TypeError", + AsymmetricPublicKeyError::InvalidPkcs1PublicKey => "TypeError", + AsymmetricPublicKeyError::AsymmetricPrivateKey(e) => { + get_asymmetric_private_key_error(e) + } + AsymmetricPublicKeyError::InvalidX509Certificate => "TypeError", + AsymmetricPublicKeyError::X509(_) => "Error", + AsymmetricPublicKeyError::X509PublicKey(e) => { + get_x509_public_key_error(e) + } + AsymmetricPublicKeyError::UnsupportedPemLabel(_) => "TypeError", + AsymmetricPublicKeyError::InvalidSpkiPublicKey => "TypeError", + AsymmetricPublicKeyError::UnsupportedKeyType(_) => "TypeError", + AsymmetricPublicKeyError::UnsupportedKeyFormat(_) => "TypeError", + AsymmetricPublicKeyError::Spki(_) => "Error", + AsymmetricPublicKeyError::Pkcs1(_) => "Error", + AsymmetricPublicKeyError::RsaPssParamsParse(_) => "TypeError", + AsymmetricPublicKeyError::MalformedDssPublicKey => "TypeError", + AsymmetricPublicKeyError::MalformedOrMissingNamedCurveInEcParameters => { + "TypeError" + } + AsymmetricPublicKeyError::MalformedOrMissingPublicKeyInEcSpki => { + "TypeError" + } + AsymmetricPublicKeyError::Ec(_) => "Error", + AsymmetricPublicKeyError::UnsupportedEcNamedCurve => "TypeError", + AsymmetricPublicKeyError::MalformedOrMissingPublicKeyInX25519Spki => { + "TypeError" + } + AsymmetricPublicKeyError::X25519PublicKeyIsTooShort => "TypeError", + AsymmetricPublicKeyError::InvalidEd25519PublicKey => "TypeError", + AsymmetricPublicKeyError::MissingDhParameters => "TypeError", + AsymmetricPublicKeyError::MalformedDhParameters => "TypeError", + AsymmetricPublicKeyError::MalformedOrMissingPublicKeyInDhSpki => { + "TypeError" + } + AsymmetricPublicKeyError::UnsupportedPrivateKeyOid => "TypeError", + } + } + + pub fn get_private_encrypt_decrypt_error( + e: &PrivateEncryptDecryptError, + ) -> &'static str { + match e { + PrivateEncryptDecryptError::Pkcs8(_) => "Error", + PrivateEncryptDecryptError::Spki(_) => "Error", + PrivateEncryptDecryptError::Utf8(_) => "Error", + PrivateEncryptDecryptError::Rsa(_) => "Error", + PrivateEncryptDecryptError::UnknownPadding => "TypeError", + } + } + + pub fn get_ecdh_encode_pub_key_error(e: &EcdhEncodePubKey) -> &'static str { + match e { + EcdhEncodePubKey::InvalidPublicKey => "TypeError", + EcdhEncodePubKey::UnsupportedCurve => "TypeError", + EcdhEncodePubKey::Sec1(_) => "Error", + } + } + + pub fn get_diffie_hellman_error(_: &DiffieHellmanError) -> &'static str { + "TypeError" + } + + pub fn get_sign_ed25519_error(_: &SignEd25519Error) -> &'static str { + "TypeError" + } + + pub fn get_verify_ed25519_error(_: &VerifyEd25519Error) -> &'static str { + "TypeError" + } +} + +fn get_os_error(error: &OsError) -> &'static str { + match error { + OsError::Permission(e) => get_permission_check_error_class(e), + OsError::InvalidUtf8(_) => "InvalidData", + OsError::EnvEmptyKey => "TypeError", + OsError::EnvInvalidKey(_) => "TypeError", + OsError::EnvInvalidValue(_) => "TypeError", + OsError::Io(e) => get_io_error_class(e), + OsError::Var(e) => get_env_var_error_class(e), + } +} + +fn get_sync_fetch_error(error: &SyncFetchError) -> &'static str { + match error { + SyncFetchError::BlobUrlsNotSupportedInContext => "TypeError", + SyncFetchError::Io(e) => get_io_error_class(e), + SyncFetchError::InvalidScriptUrl => "TypeError", + SyncFetchError::InvalidStatusCode(_) => "TypeError", + SyncFetchError::ClassicScriptSchemeUnsupportedInWorkers(_) => "TypeError", + SyncFetchError::InvalidUri(_) => "Error", + SyncFetchError::InvalidMimeType(_) => "DOMExceptionNetworkError", + SyncFetchError::MissingMimeType => "DOMExceptionNetworkError", + SyncFetchError::Fetch(e) => get_fetch_error(e), + SyncFetchError::Join(_) => "Error", + SyncFetchError::Other(e) => get_error_class_name(e).unwrap_or("Error"), + } +} + pub fn get_error_class_name(e: &AnyError) -> Option<&'static str> { deno_core::error::get_custom_error_class(e) - .or_else(|| deno_webgpu::error::get_error_class_name(e)) - .or_else(|| deno_web::get_error_class_name(e)) - .or_else(|| deno_webstorage::get_not_supported_error_class_name(e)) - .or_else(|| deno_websocket::get_network_error_class_name(e)) + .or_else(|| { + e.downcast_ref::() + .map(get_child_permission_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_permission_check_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_permission_error_class) + }) + .or_else(|| e.downcast_ref::().map(get_fs_error)) + .or_else(|| { + e.downcast_ref::() + .map(node::get_blocklist_error) + }) + .or_else(|| e.downcast_ref::().map(node::get_fs_error)) + .or_else(|| { + e.downcast_ref::() + .map(node::get_idna_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_ipc_json_stream_error) + }) + .or_else(|| e.downcast_ref::().map(node::get_ipc_error)) + .or_else(|| { + e.downcast_ref::() + .map(node::get_worker_threads_filename_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_require_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_http2_error) + }) + .or_else(|| e.downcast_ref::().map(node::get_os_error)) + .or_else(|| { + e.downcast_ref::() + .map(node::get_brotli_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_mode_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_zlib_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_crypto_cipher_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_crypto_cipher_context_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_crypto_decipher_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_crypto_decipher_context_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_x509_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_crypto_key_object_handle_prehashed_sign_and_verify_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_crypto_hash_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_asymmetric_public_key_jwk_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_generate_rsa_pss_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_asymmetric_private_key_der_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_asymmetric_public_key_der_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_export_public_key_pem_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_export_private_key_pem_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_rsa_jwk_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_ec_jwk_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_ed_raw_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_pbkdf2_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_scrypt_async_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_hkdf_error_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_rsa_pss_params_parse_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_asymmetric_private_key_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_asymmetric_public_key_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_private_encrypt_decrypt_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_ecdh_encode_pub_key_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_diffie_hellman_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_sign_ed25519_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_verify_ed25519_error) + }) + .or_else(|| e.downcast_ref::().map(get_napi_error_class)) + .or_else(|| e.downcast_ref::().map(get_web_error_class)) + .or_else(|| { + e.downcast_ref::() + .map(get_create_worker_error) + }) + .or_else(|| e.downcast_ref::().map(get_tty_error)) + .or_else(|| e.downcast_ref::().map(get_readline_error)) + .or_else(|| e.downcast_ref::().map(get_signal_error)) + .or_else(|| e.downcast_ref::().map(get_fs_events_error)) + .or_else(|| e.downcast_ref::().map(get_http_start_error)) + .or_else(|| e.downcast_ref::().map(get_process_error)) + .or_else(|| e.downcast_ref::().map(get_os_error)) + .or_else(|| e.downcast_ref::().map(get_sync_fetch_error)) + .or_else(|| { + e.downcast_ref::() + .map(get_web_compression_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_web_message_port_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_web_stream_resource_error_class) + }) + .or_else(|| e.downcast_ref::().map(get_web_blob_error_class)) + .or_else(|| e.downcast_ref::().map(|_| "TypeError")) + .or_else(|| e.downcast_ref::().map(get_ffi_repr_error_class)) + .or_else(|| e.downcast_ref::().map(get_http_error)) + .or_else(|| e.downcast_ref::().map(get_http_next_error)) + .or_else(|| { + e.downcast_ref::() + .map(get_websocket_upgrade_error) + }) + .or_else(|| e.downcast_ref::().map(get_fs_ops_error)) + .or_else(|| { + e.downcast_ref::() + .map(get_ffi_dlfcn_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_ffi_static_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_ffi_callback_error_class) + }) + .or_else(|| e.downcast_ref::().map(get_ffi_call_error_class)) + .or_else(|| e.downcast_ref::().map(get_tls_error_class)) + .or_else(|| e.downcast_ref::().map(get_cron_error_class)) + .or_else(|| e.downcast_ref::().map(get_canvas_error)) + .or_else(|| e.downcast_ref::().map(get_cache_error)) + .or_else(|| e.downcast_ref::().map(get_websocket_error)) + .or_else(|| { + e.downcast_ref::() + .map(get_websocket_handshake_error) + }) + .or_else(|| e.downcast_ref::().map(get_kv_error)) + .or_else(|| e.downcast_ref::().map(get_fetch_error)) + .or_else(|| { + e.downcast_ref::() + .map(get_http_client_create_error) + }) + .or_else(|| e.downcast_ref::().map(get_net_error)) + .or_else(|| { + e.downcast_ref::() + .map(get_net_map_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_broadcast_channel_error) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_webgpu_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_webgpu_buffer_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_webgpu_bundle_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_webgpu_byow_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_webgpu_render_pass_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_webgpu_surface_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_crypto_decrypt_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_crypto_encrypt_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_crypto_shared_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_crypto_ed25519_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_crypto_export_key_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_crypto_generate_key_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_crypto_import_key_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_crypto_x448_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_crypto_x25519_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_crypto_error_class) + }) + .or_else(|| { + e.downcast_ref::() + .map(get_webstorage_class_name) + }) + .or_else(|| { + e.downcast_ref::() + .map(|_| "TypeError") + }) .or_else(|| { e.downcast_ref::() .map(get_dlopen_error_class) diff --git a/runtime/examples/extension/main.rs b/runtime/examples/extension/main.rs index 6f4f02508b6bcf..1ff16ec83f69b1 100644 --- a/runtime/examples/extension/main.rs +++ b/runtime/examples/extension/main.rs @@ -16,6 +16,7 @@ use deno_runtime::deno_permissions::PermissionsContainer; use deno_runtime::permissions::RuntimePermissionDescriptorParser; use deno_runtime::worker::MainWorker; use deno_runtime::worker::WorkerOptions; +use deno_runtime::worker::WorkerServiceOptions; #[op2(fast)] fn op_hello(#[string] text: &str) { @@ -35,13 +36,27 @@ async fn main() -> Result<(), AnyError> { Path::new(env!("CARGO_MANIFEST_DIR")).join("examples/extension/main.js"); let main_module = ModuleSpecifier::from_file_path(js_path).unwrap(); eprintln!("Running {main_module}..."); + let fs = Arc::new(RealFs); + let permission_desc_parser = + Arc::new(RuntimePermissionDescriptorParser::new(fs.clone())); let mut worker = MainWorker::bootstrap_from_options( main_module.clone(), - PermissionsContainer::allow_all(Arc::new( - RuntimePermissionDescriptorParser::new(Arc::new(RealFs)), - )), - WorkerOptions { + WorkerServiceOptions { module_loader: Rc::new(FsModuleLoader), + permissions: PermissionsContainer::allow_all(permission_desc_parser), + blob_store: Default::default(), + broadcast_channel: Default::default(), + feature_checker: Default::default(), + node_services: Default::default(), + npm_process_state_provider: Default::default(), + root_cert_store_provider: Default::default(), + fetch_dns_resolver: Default::default(), + shared_array_buffer_store: Default::default(), + compiled_wasm_module_store: Default::default(), + v8_code_cache: Default::default(), + fs, + }, + WorkerOptions { extensions: vec![hello_runtime::init_ops_and_esm()], ..Default::default() }, diff --git a/runtime/fmt_errors.rs b/runtime/fmt_errors.rs index b2cec2a5a6e1d5..6f120b5d4669bd 100644 --- a/runtime/fmt_errors.rs +++ b/runtime/fmt_errors.rs @@ -1,11 +1,10 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. //! This mod provides DenoError to unify errors across Deno. +use color_print::cformat; +use color_print::cstr; use deno_core::error::format_frame; use deno_core::error::JsError; -use deno_terminal::colors::cyan; -use deno_terminal::colors::italic_bold; -use deno_terminal::colors::red; -use deno_terminal::colors::yellow; +use deno_terminal::colors; use std::fmt::Write as _; #[derive(Debug, Clone)] @@ -24,26 +23,54 @@ struct IndexedErrorReference<'a> { enum FixSuggestionKind { Info, Hint, + Docs, +} + +#[derive(Debug)] +enum FixSuggestionMessage<'a> { + Single(&'a str), + Multiline(&'a [&'a str]), } #[derive(Debug)] pub struct FixSuggestion<'a> { kind: FixSuggestionKind, - message: &'a str, + message: FixSuggestionMessage<'a>, } impl<'a> FixSuggestion<'a> { pub fn info(message: &'a str) -> Self { Self { kind: FixSuggestionKind::Info, - message, + message: FixSuggestionMessage::Single(message), + } + } + + pub fn info_multiline(messages: &'a [&'a str]) -> Self { + Self { + kind: FixSuggestionKind::Info, + message: FixSuggestionMessage::Multiline(messages), } } pub fn hint(message: &'a str) -> Self { Self { kind: FixSuggestionKind::Hint, - message, + message: FixSuggestionMessage::Single(message), + } + } + + pub fn hint_multiline(messages: &'a [&'a str]) -> Self { + Self { + kind: FixSuggestionKind::Hint, + message: FixSuggestionMessage::Multiline(messages), + } + } + + pub fn docs(url: &'a str) -> Self { + Self { + kind: FixSuggestionKind::Docs, + message: FixSuggestionMessage::Single(url), } } } @@ -58,10 +85,10 @@ impl deno_core::error::ErrorFormat for AnsiColors { use deno_core::error::ErrorElement::*; match element { Anonymous | NativeFrame | FileName | EvalOrigin => { - cyan(s).to_string().into() + colors::cyan(s).to_string().into() } - LineNumber | ColumnNumber => yellow(s).to_string().into(), - FunctionName | PromiseAll => italic_bold(s).to_string().into(), + LineNumber | ColumnNumber => colors::yellow(s).to_string().into(), + FunctionName | PromiseAll => colors::italic_bold(s).to_string().into(), } } } @@ -94,7 +121,7 @@ fn format_maybe_source_line( if column_number as usize > source_line.len() { return format!( "\n{} Couldn't format source line: Column {} is out of bounds (source may have changed at runtime)", - yellow("Warning"), column_number, + colors::yellow("Warning"), column_number, ); } @@ -107,9 +134,9 @@ fn format_maybe_source_line( } s.push('^'); let color_underline = if is_error { - red(&s).to_string() + colors::red(&s).to_string() } else { - cyan(&s).to_string() + colors::cyan(&s).to_string() }; let indent = format!("{:indent$}", "", indent = level); @@ -180,7 +207,8 @@ fn format_js_error_inner( if let Some(circular) = &circular { if js_error.is_same_error(circular.reference.to) { - write!(s, " {}", cyan(format!("", circular.index))).unwrap(); + write!(s, " {}", colors::cyan(format!("", circular.index))) + .unwrap(); } } @@ -218,7 +246,8 @@ fn format_js_error_inner( .unwrap_or(false); let error_string = if is_caused_by_circular { - cyan(format!("[Circular *{}]", circular.unwrap().index)).to_string() + colors::cyan(format!("[Circular *{}]", circular.unwrap().index)) + .to_string() } else { format_js_error_inner(cause, circular, false, vec![]) }; @@ -235,10 +264,35 @@ fn format_js_error_inner( for (index, suggestion) in suggestions.iter().enumerate() { write!(s, " ").unwrap(); match suggestion.kind { - FixSuggestionKind::Hint => write!(s, "{} ", cyan("hint:")).unwrap(), - FixSuggestionKind::Info => write!(s, "{} ", yellow("info:")).unwrap(), + FixSuggestionKind::Hint => { + write!(s, "{} ", colors::cyan("hint:")).unwrap() + } + FixSuggestionKind::Info => { + write!(s, "{} ", colors::yellow("info:")).unwrap() + } + FixSuggestionKind::Docs => { + write!(s, "{} ", colors::green("docs:")).unwrap() + } }; - write!(s, "{}", suggestion.message).unwrap(); + match suggestion.message { + FixSuggestionMessage::Single(msg) => { + if matches!(suggestion.kind, FixSuggestionKind::Docs) { + write!(s, "{}", cformat!("{}", msg)).unwrap(); + } else { + write!(s, "{}", msg).unwrap(); + } + } + FixSuggestionMessage::Multiline(messages) => { + for (idx, message) in messages.iter().enumerate() { + if idx != 0 { + writeln!(s).unwrap(); + write!(s, " ").unwrap(); + } + write!(s, "{}", message).unwrap(); + } + } + } + if index != (suggestions.len() - 1) { writeln!(s).unwrap(); } @@ -248,28 +302,175 @@ fn format_js_error_inner( s } -/// Format a [`JsError`] for terminal output. -pub fn format_js_error(js_error: &JsError) -> String { - let circular = - find_recursive_cause(js_error).map(|reference| IndexedErrorReference { - reference, - index: 1, - }); +fn get_suggestions_for_terminal_errors(e: &JsError) -> Vec { + if let Some(msg) = &e.message { + if msg.contains("module is not defined") + || msg.contains("exports is not defined") + || msg.contains("require is not defined") + { + return vec![ + FixSuggestion::info_multiline(&[ + cstr!("Deno supports CommonJS modules in .cjs files, or when the closest"), + cstr!("package.json has a \"type\": \"commonjs\" option.") + ]), + FixSuggestion::hint_multiline(&[ + "Rewrite this module to ESM,", + cstr!("or change the file extension to .cjs,"), + cstr!("or add package.json next to the file with \"type\": \"commonjs\" option,"), + cstr!("or pass --unstable-detect-cjs flag to detect CommonJS when loading."), + ]), + FixSuggestion::docs("https://docs.deno.com/go/commonjs"), + ]; + } else if msg.contains("__filename is not defined") { + return vec![ + FixSuggestion::info(cstr!( + "__filename global is not available in ES modules." + )), + FixSuggestion::hint(cstr!("Use import.meta.filename instead.")), + ]; + } else if msg.contains("__dirname is not defined") { + return vec![ + FixSuggestion::info(cstr!( + "__dirname global is not available in ES modules." + )), + FixSuggestion::hint(cstr!("Use import.meta.dirname instead.")), + ]; + } else if msg.contains("Buffer is not defined") { + return vec![ + FixSuggestion::info(cstr!( + "Buffer is not available in the global scope in Deno." + )), + FixSuggestion::hint_multiline(&[ + cstr!("Import it explicitly with import { Buffer } from \"node:buffer\";,"), + cstr!("or run again with --unstable-node-globals flag to add this global."), + ]), + ]; + } else if msg.contains("clearImmediate is not defined") { + return vec![ + FixSuggestion::info(cstr!( + "clearImmediate is not available in the global scope in Deno." + )), + FixSuggestion::hint_multiline(&[ + cstr!("Import it explicitly with import { clearImmediate } from \"node:timers\";,"), + cstr!("or run again with --unstable-node-globals flag to add this global."), + ]), + ]; + } else if msg.contains("setImmediate is not defined") { + return vec![ + FixSuggestion::info(cstr!( + "setImmediate is not available in the global scope in Deno." + )), + FixSuggestion::hint_multiline( + &[cstr!("Import it explicitly with import { setImmediate } from \"node:timers\";,"), + cstr!("or run again with --unstable-node-globals flag to add this global."), + ]), + ]; + } else if msg.contains("global is not defined") { + return vec![ + FixSuggestion::info(cstr!( + "global is not available in the global scope in Deno." + )), + FixSuggestion::hint_multiline(&[ + cstr!("Use globalThis instead, or assign globalThis.global = globalThis,"), + cstr!("or run again with --unstable-node-globals flag to add this global."), + ]), + ]; + } else if msg.contains("openKv is not a function") { + return vec![ + FixSuggestion::info("Deno.openKv() is an unstable API."), + FixSuggestion::hint( + "Run again with `--unstable-kv` flag to enable this API.", + ), + ]; + } else if msg.contains("cron is not a function") { + return vec![ + FixSuggestion::info("Deno.cron() is an unstable API."), + FixSuggestion::hint( + "Run again with `--unstable-cron` flag to enable this API.", + ), + ]; + } else if msg.contains("WebSocketStream is not defined") { + return vec![ + FixSuggestion::info("new WebSocketStream() is an unstable API."), + FixSuggestion::hint( + "Run again with `--unstable-net` flag to enable this API.", + ), + ]; + } else if msg.contains("Temporal is not defined") { + return vec![ + FixSuggestion::info("Temporal is an unstable API."), + FixSuggestion::hint( + "Run again with `--unstable-temporal` flag to enable this API.", + ), + ]; + } else if msg.contains("BroadcastChannel is not defined") { + return vec![ + FixSuggestion::info("BroadcastChannel is an unstable API."), + FixSuggestion::hint( + "Run again with `--unstable-broadcast-channel` flag to enable this API.", + ), + ]; + } else if msg.contains("window is not defined") { + return vec![ + FixSuggestion::info("window global is not available in Deno 2."), + FixSuggestion::hint("Replace `window` with `globalThis`."), + ]; + } else if msg.contains("UnsafeWindowSurface is not a constructor") { + return vec![ + FixSuggestion::info("Deno.UnsafeWindowSurface is an unstable API."), + FixSuggestion::hint( + "Run again with `--unstable-webgpu` flag to enable this API.", + ), + ]; + // Try to capture errors like: + // ``` + // Uncaught Error: Cannot find module '../build/Release/canvas.node' + // Require stack: + // - /.../deno/npm/registry.npmjs.org/canvas/2.11.2/lib/bindings.js + // - /.../.cache/deno/npm/registry.npmjs.org/canvas/2.11.2/lib/canvas.js + // ``` + } else if msg.contains("Cannot find module") + && msg.contains("Require stack") + && msg.contains(".node'") + { + return vec![ + FixSuggestion::info_multiline( + &[ + "Trying to execute an npm package using Node-API addons,", + "these packages require local `node_modules` directory to be present." + ] + ), + FixSuggestion::hint_multiline( + &[ + "Add `\"nodeModulesDir\": \"auto\" option to `deno.json`, and then run", + "`deno install --allow-scripts=npm: --entrypoint ``` + + +```sql + seLect * , biz, buz +from baz; +``` + +```sql +-- deno-fmt-ignore-file + seLect * , biz, buz +from baz; +``` diff --git a/tests/testdata/fmt/badly_formatted.sql b/tests/testdata/fmt/badly_formatted.sql new file mode 100644 index 00000000000000..8b4cb978f0c896 --- /dev/null +++ b/tests/testdata/fmt/badly_formatted.sql @@ -0,0 +1,21 @@ +select * from foo; +update foo set a = 'b'Where id = 'biz'; + + + create table foo(id text not null +bar text, + biz int, + buz number NOT NULL +); + +INSERT + into + user_data + (first_name, +last_name, address, phone, email) +VALUES + ('foo', 'bar', + 'biz', 1, 'bix'); + + + diff --git a/tests/testdata/fmt/badly_formatted_fixed.md b/tests/testdata/fmt/badly_formatted_fixed.md index 21176742bbdf7a..7a482e058f632e 100644 --- a/tests/testdata/fmt/badly_formatted_fixed.md +++ b/tests/testdata/fmt/badly_formatted_fixed.md @@ -56,3 +56,18 @@ function foo(): number { let a: number; ``` + +```sql +SELECT + *, + biz, + buz +FROM + baz; +``` + +```sql +-- deno-fmt-ignore-file + seLect * , biz, buz +from baz; +``` diff --git a/tests/testdata/fmt/badly_formatted_fixed.sql b/tests/testdata/fmt/badly_formatted_fixed.sql new file mode 100644 index 00000000000000..d50c6192161949 --- /dev/null +++ b/tests/testdata/fmt/badly_formatted_fixed.sql @@ -0,0 +1,22 @@ +SELECT + * +FROM + foo; + +UPDATE + foo +SET + a = 'b' +WHERE + id = 'biz'; + +CREATE TABLE foo( + id text NOT NULL bar text, + biz int, + buz number NOT NULL +); + +INSERT INTO + user_data (first_name, last_name, address, phone, email) +VALUES + ('foo', 'bar', 'biz', 1, 'bix'); diff --git a/tests/testdata/fmt/expected_fmt_check_verbose_formatted_files.out b/tests/testdata/fmt/expected_fmt_check_verbose_formatted_files.out deleted file mode 100644 index 158c556c2968ff..00000000000000 --- a/tests/testdata/fmt/expected_fmt_check_verbose_formatted_files.out +++ /dev/null @@ -1 +0,0 @@ -Checked 2 files diff --git a/tests/testdata/fmt/fmt_with_config_default.out b/tests/testdata/fmt/fmt_with_config_default.out deleted file mode 100644 index faad9352bbb63b..00000000000000 --- a/tests/testdata/fmt/fmt_with_config_default.out +++ /dev/null @@ -1,2 +0,0 @@ -Config file found at '[WILDCARD]deno.jsonc' -Checked 2 files diff --git a/tests/testdata/npm/check_errors/main_all.out b/tests/testdata/npm/check_errors/main_all.out deleted file mode 100644 index 4c624c0eaa9a26..00000000000000 --- a/tests/testdata/npm/check_errors/main_all.out +++ /dev/null @@ -1,19 +0,0 @@ -Download http://localhost:4260/@denotest/check-error -Download http://localhost:4260/@denotest/check-error/1.0.0.tgz -Check file:///[WILDCARD]/check_errors/main.ts -error: TS2506 [ERROR]: 'Class1' is referenced directly or indirectly in its own base expression. -export class Class1 extends Class2 { - ~~~~~~ - at file:///[WILDCARD]/check-error/1.0.0/index.d.ts:2:14 - -TS2506 [ERROR]: 'Class2' is referenced directly or indirectly in its own base expression. -export class Class2 extends Class1 { - ~~~~~~ - at file:///[WILDCARD]/check-error/1.0.0/index.d.ts:5:14 - -TS2339 [ERROR]: Property 'Asdf' does not exist on type 'typeof import("file:///[WILDCARD]/@denotest/check-error/1.0.0/index.d.ts")'. -console.log(test.Asdf); // should error - ~~~~ - at file:///[WILDCARD]/check_errors/main.ts:3:18 - -Found 3 errors. diff --git a/tests/testdata/npm/check_errors/main_local.out b/tests/testdata/npm/check_errors/main_local.out deleted file mode 100644 index 4d3a892e74a8b7..00000000000000 --- a/tests/testdata/npm/check_errors/main_local.out +++ /dev/null @@ -1,7 +0,0 @@ -Download http://localhost:4260/@denotest/check-error -Download http://localhost:4260/@denotest/check-error/1.0.0.tgz -Check file:///[WILDCARD]/check_errors/main.ts -error: TS2339 [ERROR]: Property 'Asdf' does not exist on type 'typeof import("file:///[WILDCARD]/@denotest/check-error/1.0.0/index.d.ts")'. -console.log(test.Asdf); // should error - ~~~~ - at file:///[WILDCARD]/npm/check_errors/main.ts:3:18 diff --git a/tests/testdata/npm/cjs_local_global_decls/main.out b/tests/testdata/npm/cjs_local_global_decls/main.out deleted file mode 100644 index 5e7a36c8d0410a..00000000000000 --- a/tests/testdata/npm/cjs_local_global_decls/main.out +++ /dev/null @@ -1,3 +0,0 @@ -Download http://localhost:4260/@denotest/cjs-local-global-decls -Download http://localhost:4260/@denotest/cjs-local-global-decls/1.0.0.tgz -Loaded. diff --git a/tests/testdata/npm/cjs_pkg_imports/main.out b/tests/testdata/npm/cjs_pkg_imports/main.out deleted file mode 100644 index 661146bd036c4e..00000000000000 --- a/tests/testdata/npm/cjs_pkg_imports/main.out +++ /dev/null @@ -1,3 +0,0 @@ -Download http://localhost:4260/@denotest/cjs-pkg-imports -Download http://localhost:4260/@denotest/cjs-pkg-imports/1.0.0.tgz -{ crypto: Crypto { subtle: SubtleCrypto {} }, number: 5 } diff --git a/tests/testdata/npm/cjs_this_in_exports/main.out b/tests/testdata/npm/cjs_this_in_exports/main.out deleted file mode 100644 index ba436bddc90a62..00000000000000 --- a/tests/testdata/npm/cjs_this_in_exports/main.out +++ /dev/null @@ -1,5 +0,0 @@ -1 -1 -error: Uncaught (in promise) TypeError: this.otherMethod is not a function - at getValue (file://[WILDCARD]/@denotest/cjs-this-in-exports/1.0.0/index.js:3:17) - at file://[WILDCARD]/testdata/npm/cjs_this_in_exports/main.js:11:1 diff --git a/tests/testdata/npm/cjs_with_deps/main_info_json.out b/tests/testdata/npm/cjs_with_deps/main_info_json.out index af1ef1351880a7..137b9f8ce56c3d 100644 --- a/tests/testdata/npm/cjs_with_deps/main_info_json.out +++ b/tests/testdata/npm/cjs_with_deps/main_info_json.out @@ -1,4 +1,5 @@ { + "version": 1, "roots": [ "file://[WILDCARD]/main.js" ], @@ -51,7 +52,6 @@ "npm:chai@4.3": "npm:/chai@4.3.6", "npm:chalk@4": "npm:/chalk@4.1.2" }, - "version": 1, "npmPackages": { "ansi-styles@4.3.0": { "name": "ansi-styles", diff --git a/tests/testdata/npm/compare_globals/main.out b/tests/testdata/npm/compare_globals/main.out deleted file mode 100644 index 234a68971fa538..00000000000000 --- a/tests/testdata/npm/compare_globals/main.out +++ /dev/null @@ -1,30 +0,0 @@ -[UNORDERED_START] -Download http://localhost:4260/@types/node -Download http://localhost:4260/undici-types -Download http://localhost:4260/@denotest/globals -[UNORDERED_END] -[UNORDERED_START] -Download http://localhost:4260/@denotest/globals/1.0.0.tgz -Download http://localhost:4260/@types/node/node-22.5.4.tgz -Download http://localhost:4260/undici-types/undici-types-6.19.8.tgz -[UNORDERED_END] -Check file:///[WILDCARD]/npm/compare_globals/main.ts -true -true -[] -setTimeout 1 false -setTimeout 2 function -setTimeout 3 function -setTimeout 4 function -setTimeout 5 undefined -window 1 false -window 2 false -false -false -self 1 true -self 2 true -false -false -bar -bar -true diff --git a/tests/testdata/npm/compare_globals/main.ts b/tests/testdata/npm/compare_globals/main.ts deleted file mode 100644 index 9482798d8c9a24..00000000000000 --- a/tests/testdata/npm/compare_globals/main.ts +++ /dev/null @@ -1,54 +0,0 @@ -/// - -import * as globals from "npm:@denotest/globals"; -console.log(globals.global === globals.globalThis); -// @ts-expect-error even though these are the same object, they have different types -console.log(globals.globalThis === globalThis); -console.log(globals.process.execArgv); - -type AssertTrue = never; -type _TestNoProcessGlobal = AssertTrue< - typeof globalThis extends { process: any } ? false : true ->; -type _TestHasNodeJsGlobal = NodeJS.Architecture; - -const controller = new AbortController(); -controller.abort("reason"); // in the NodeJS declaration it doesn't have a reason - -// Some globals are not the same between Node and Deno. -// @ts-expect-error incompatible types between Node and Deno -console.log("setTimeout 1", globalThis.setTimeout === globals.getSetTimeout()); - -// Super edge case where some Node code deletes a global where the -// Node code has its own global and the Deno code has the same global, -// but it's different. Basically if some Node code deletes -// one of these globals then we don't want it to suddenly inherit -// the Deno global (or touch the Deno global at all). -console.log("setTimeout 2", typeof globalThis.setTimeout); -console.log("setTimeout 3", typeof globals.getSetTimeout()); -globals.deleteSetTimeout(); -console.log("setTimeout 4", typeof globalThis.setTimeout); -console.log("setTimeout 5", typeof globals.getSetTimeout()); - -// In Deno 2 and Node.js, the window global is not defined. -console.log("window 1", "window" in globalThis); -console.log( - "window 2", - Object.getOwnPropertyDescriptor(globalThis, "window") !== undefined, -); -globals.checkWindowGlobal(); - -// In Deno 2 self global is defined, but in Node it is not. -console.log("self 1", "self" in globalThis); -console.log( - "self 2", - Object.getOwnPropertyDescriptor(globalThis, "self") !== undefined, -); -globals.checkSelfGlobal(); - -// "Non-managed" globals are shared between Node and Deno. -(globalThis as any).foo = "bar"; -console.log((globalThis as any).foo); -console.log(globals.getFoo()); - -console.log(Reflect.ownKeys(globalThis).includes("console")); // non-enumerable keys are included diff --git a/tests/testdata/npm/d_ext/main.out b/tests/testdata/npm/d_ext/main.out deleted file mode 100644 index 5c22a76a0cb204..00000000000000 --- a/tests/testdata/npm/d_ext/main.out +++ /dev/null @@ -1,3 +0,0 @@ -Download http://localhost:4260/@denotest/d-ext -Download http://localhost:4260/@denotest/d-ext/1.0.0.tgz -Check file:///[WILDCARD]/npm/d_ext/main.ts diff --git a/tests/testdata/npm/deno_run_cowsay_no_permissions.out b/tests/testdata/npm/deno_run_cowsay_no_permissions.out index 6434620e2e58c6..25b79d9a7dce23 100644 --- a/tests/testdata/npm/deno_run_cowsay_no_permissions.out +++ b/tests/testdata/npm/deno_run_cowsay_no_permissions.out @@ -1,2 +1,2 @@ -error: Uncaught NotCapable: Requires read access to , specify the required permissions during compilation using `deno compile --allow-read` +error: Uncaught (in promise) NotCapable: Requires read access to , specify the required permissions during compilation using `deno compile --allow-read` [WILDCARD] diff --git a/tests/testdata/npm/error_version_after_subpath/main.out b/tests/testdata/npm/error_version_after_subpath/main.out deleted file mode 100644 index 0cdd1b6da01fd1..00000000000000 --- a/tests/testdata/npm/error_version_after_subpath/main.out +++ /dev/null @@ -1,2 +0,0 @@ -error: Invalid package specifier 'npm:react-dom/server@18.2.0'. Did you mean to write 'npm:react-dom@18.2.0/server'? - at [WILDCARD]/npm/error_version_after_subpath/main.js:1:8 diff --git a/tests/testdata/npm/file_dts_dmts_dcts/main.out b/tests/testdata/npm/file_dts_dmts_dcts/main.out deleted file mode 100644 index 507d2c2f79944f..00000000000000 --- a/tests/testdata/npm/file_dts_dmts_dcts/main.out +++ /dev/null @@ -1,24 +0,0 @@ -Download http://localhost:4260/@denotest/file-dts-dmts-dcts -Download http://localhost:4260/@denotest/file-dts-dmts-dcts/1.0.0.tgz -Check file:///[WILDCARD]/main.ts -error: TS2322 [ERROR]: Type '5' is not assignable to type '"dts"'. -const value1: Dts1 = 5; - ~~~~~~ - at file:///[WILDCARD] - -TS2322 [ERROR]: Type '5' is not assignable to type '"mts"'. -const value2: Mts1 = 5; - ~~~~~~ - at file:///[WILDCARD] - -TS2322 [ERROR]: Type '5' is not assignable to type '"mts"'. -const value3: Mts2 = 5; - ~~~~~~ - at file:///[WILDCARD] - -TS2322 [ERROR]: Type '5' is not assignable to type '"cts"'. -const value4: Cts1 = 5; - ~~~~~~ - at file:///[WILDCARD] - -Found 4 errors. diff --git a/tests/testdata/npm/import_map/main.out b/tests/testdata/npm/import_map/main.out deleted file mode 100644 index 05f313d4e99f14..00000000000000 --- a/tests/testdata/npm/import_map/main.out +++ /dev/null @@ -1,10 +0,0 @@ -[UNORDERED_START] -Download http://localhost:4260/chalk -Download http://localhost:4260/@denotest/dual-cjs-esm -[UNORDERED_END] -[UNORDERED_START] -Download http://localhost:4260/@denotest/dual-cjs-esm/1.0.0.tgz -Download http://localhost:4260/chalk/chalk-5.0.1.tgz -[UNORDERED_END] -chalk import map loads -esm diff --git a/tests/testdata/npm/imports_package_json/main.out b/tests/testdata/npm/imports_package_json/main.out deleted file mode 100644 index 979e355765e73b..00000000000000 --- a/tests/testdata/npm/imports_package_json/main.out +++ /dev/null @@ -1,7 +0,0 @@ -Download http://localhost:4260/@denotest/imports-package-json -Download http://localhost:4260/@denotest/imports-package-json/1.0.0.tgz -hi -bye -function -function -function diff --git a/tests/testdata/npm/info/chalk_json.out b/tests/testdata/npm/info/chalk_json.out deleted file mode 100644 index d54155270b1679..00000000000000 --- a/tests/testdata/npm/info/chalk_json.out +++ /dev/null @@ -1,57 +0,0 @@ -{ - "roots": [ - "npm:chalk@4" - ], - "modules": [ - { - "kind": "npm", - "specifier": "npm:/chalk@4.1.2", - "npmPackage": "chalk@4.1.2" - } - ], - "redirects": { - "npm:chalk@4": "npm:/chalk@4.1.2" - }, - "version": 1, - "npmPackages": { - "ansi-styles@4.3.0": { - "name": "ansi-styles", - "version": "4.3.0", - "dependencies": [ - "color-convert@2.0.1" - ] - }, - "chalk@4.1.2": { - "name": "chalk", - "version": "4.1.2", - "dependencies": [ - "ansi-styles@4.3.0", - "supports-color@7.2.0" - ] - }, - "color-convert@2.0.1": { - "name": "color-convert", - "version": "2.0.1", - "dependencies": [ - "color-name@1.1.4" - ] - }, - "color-name@1.1.4": { - "name": "color-name", - "version": "1.1.4", - "dependencies": [] - }, - "has-flag@4.0.0": { - "name": "has-flag", - "version": "4.0.0", - "dependencies": [] - }, - "supports-color@7.2.0": { - "name": "supports-color", - "version": "7.2.0", - "dependencies": [ - "has-flag@4.0.0" - ] - } - } -} diff --git a/tests/testdata/npm/mixed_case_package_name/global.out b/tests/testdata/npm/mixed_case_package_name/global.out deleted file mode 100644 index fdacea3852735e..00000000000000 --- a/tests/testdata/npm/mixed_case_package_name/global.out +++ /dev/null @@ -1,9 +0,0 @@ -[UNORDERED_START] -Download http://localhost:4260/@denotest/MixedCase -Download http://localhost:4260/@denotest/CAPITALS -[UNORDERED_END] -[UNORDERED_START] -Download http://localhost:4260/@denotest/CAPITALS/1.0.0.tgz -Download http://localhost:4260/@denotest/MixedCase/1.0.0.tgz -[UNORDERED_END] -5 diff --git a/tests/testdata/npm/mixed_case_package_name/local.out b/tests/testdata/npm/mixed_case_package_name/local.out deleted file mode 100644 index 6ab989d80c3d13..00000000000000 --- a/tests/testdata/npm/mixed_case_package_name/local.out +++ /dev/null @@ -1,13 +0,0 @@ -[UNORDERED_START] -Download http://localhost:4260/@denotest/MixedCase -Download http://localhost:4260/@denotest/CAPITALS -[UNORDERED_END] -[UNORDERED_START] -Download http://localhost:4260/@denotest/CAPITALS/1.0.0.tgz -Initialize @denotest/CAPITALS@1.0.0 -Download http://localhost:4260/@denotest/MixedCase/1.0.0.tgz -Initialize @denotest/MixedCase@1.0.0 -[UNORDERED_END] -5 -true -true diff --git a/tests/testdata/npm/peer_deps_with_copied_folders/main.out b/tests/testdata/npm/peer_deps_with_copied_folders/main.out index 3c133bcde42a3b..b7a5835577b216 100644 --- a/tests/testdata/npm/peer_deps_with_copied_folders/main.out +++ b/tests/testdata/npm/peer_deps_with_copied_folders/main.out @@ -1,7 +1,7 @@ [UNORDERED_START] -Download http://localhost:4260/@denotest/peer-dep-test-child -Download http://localhost:4260/@denotest/peer-dep-test-grandchild -Download http://localhost:4260/@denotest/peer-dep-test-peer +Download http://localhost:4260/@denotest%2fpeer-dep-test-child +Download http://localhost:4260/@denotest%2fpeer-dep-test-grandchild +Download http://localhost:4260/@denotest%2fpeer-dep-test-peer [UNORDERED_END] [UNORDERED_START] Download http://localhost:4260/@denotest/peer-dep-test-child/1.0.0.tgz diff --git a/tests/testdata/npm/peer_deps_with_copied_folders/main_info_json.out b/tests/testdata/npm/peer_deps_with_copied_folders/main_info_json.out index df6541aa8e83ad..d87496d6d54b6d 100644 --- a/tests/testdata/npm/peer_deps_with_copied_folders/main_info_json.out +++ b/tests/testdata/npm/peer_deps_with_copied_folders/main_info_json.out @@ -1,4 +1,5 @@ { + "version": 1, "roots": [ "[WILDCARD]/npm/peer_deps_with_copied_folders/main.ts" ], @@ -51,7 +52,6 @@ "npm:@denotest/peer-dep-test-child@1": "npm:/@denotest/peer-dep-test-child@1.0.0", "npm:@denotest/peer-dep-test-child@2": "npm:/@denotest/peer-dep-test-child@2.0.0" }, - "version": 1, "npmPackages": { "@denotest/peer-dep-test-child@1.0.0_@denotest+peer-dep-test-peer@1.0.0": { "name": "@denotest/peer-dep-test-child", diff --git a/tests/testdata/npm/peer_deps_with_copied_folders/main_node_modules_reload.out b/tests/testdata/npm/peer_deps_with_copied_folders/main_node_modules_reload.out index 9c8145211899e1..18d7f7865ba6f0 100644 --- a/tests/testdata/npm/peer_deps_with_copied_folders/main_node_modules_reload.out +++ b/tests/testdata/npm/peer_deps_with_copied_folders/main_node_modules_reload.out @@ -1,7 +1,7 @@ [UNORDERED_START] -Download http://localhost:4260/@denotest/peer-dep-test-child -Download http://localhost:4260/@denotest/peer-dep-test-grandchild -Download http://localhost:4260/@denotest/peer-dep-test-peer +Download http://localhost:4260/@denotest%2fpeer-dep-test-child +Download http://localhost:4260/@denotest%2fpeer-dep-test-grandchild +Download http://localhost:4260/@denotest%2fpeer-dep-test-peer [UNORDERED_END] [UNORDERED_START] Download http://localhost:4260/@denotest/peer-dep-test-child/1.0.0.tgz diff --git a/tests/testdata/npm/permissions_outside_package/foo/package.json b/tests/testdata/npm/permissions_outside_package/foo/package.json deleted file mode 100644 index cc049e6ce909c0..00000000000000 --- a/tests/testdata/npm/permissions_outside_package/foo/package.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "foobar", - "version": "0.0.1" -} diff --git a/tests/testdata/npm/permissions_outside_package/main.out b/tests/testdata/npm/permissions_outside_package/main.out deleted file mode 100644 index 089f329c4f7581..00000000000000 --- a/tests/testdata/npm/permissions_outside_package/main.out +++ /dev/null @@ -1,3 +0,0 @@ -Download http://localhost:4260/@denotest/permissions-outside-package -Download http://localhost:4260/@denotest/permissions-outside-package/1.0.0.tgz -{ name: "foobar", version: "0.0.1" } diff --git a/tests/testdata/npm/permissions_outside_package/main.ts b/tests/testdata/npm/permissions_outside_package/main.ts deleted file mode 100644 index b0b82b6260cc8c..00000000000000 --- a/tests/testdata/npm/permissions_outside_package/main.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { loadConfigFile } from "npm:@denotest/permissions-outside-package"; - -const fileName = `${Deno.cwd()}/npm/permissions_outside_package/foo/config.js`; -const config = loadConfigFile(fileName); -console.log(config); diff --git a/tests/testdata/npm/remote_npm_specifier/main.ts b/tests/testdata/npm/remote_npm_specifier/main.ts deleted file mode 100644 index 20a99b68839595..00000000000000 --- a/tests/testdata/npm/remote_npm_specifier/main.ts +++ /dev/null @@ -1 +0,0 @@ -import "http://localhost:4545/npm/remote_npm_specifier/remote.ts"; diff --git a/tests/testdata/npm/reserved_word_exports/main.out b/tests/testdata/npm/reserved_word_exports/main.out deleted file mode 100644 index ccfd35bc862928..00000000000000 --- a/tests/testdata/npm/reserved_word_exports/main.out +++ /dev/null @@ -1,141 +0,0 @@ -Download http://localhost:4260/@denotest/reserved-word-exports -Download http://localhost:4260/@denotest/reserved-word-exports/1.0.0.tgz -[Module: null prototype] { - abstract: "abstract", - arguments: "arguments", - async: "async", - await: "await", - boolean: "boolean", - break: "break", - byte: "byte", - case: "case", - catch: "catch", - char: "char", - class: "class", - const: "const", - continue: "continue", - debugger: "debugger", - default: { - abstract: "abstract", - arguments: "arguments", - async: "async", - await: "await", - boolean: "boolean", - break: "break", - byte: "byte", - case: "case", - catch: "catch", - char: "char", - class: "class", - const: "const", - continue: "continue", - debugger: "debugger", - default: "default", - delete: "delete", - do: "do", - double: "double", - else: "else", - enum: "enum", - eval: "eval", - export: "export", - extends: "extends", - false: "false", - final: "final", - finally: "finally", - float: "float", - for: "for", - function: "function", - get: "get", - goto: "goto", - if: "if", - implements: "implements", - import: "import", - in: "in", - instanceof: "instanceof", - int: "int", - interface: "interface", - let: "let", - long: "long", - mod: "mod", - native: "native", - new: "new", - null: "null", - package: "package", - private: "private", - protected: "protected", - public: "public", - return: "return", - set: "set", - short: "short", - static: "static", - super: "super", - switch: "switch", - synchronized: "synchronized", - this: "this", - throw: "throw", - throws: "throws", - transient: "transient", - true: "true", - try: "try", - typeof: "typeof", - var: "var", - void: "void", - volatile: "volatile", - while: "while", - with: "with", - yield: "yield" - }, - delete: "delete", - do: "do", - double: "double", - else: "else", - enum: "enum", - eval: "eval", - export: "export", - extends: "extends", - false: "false", - final: "final", - finally: "finally", - float: "float", - for: "for", - function: "function", - get: "get", - goto: "goto", - if: "if", - implements: "implements", - import: "import", - in: "in", - instanceof: "instanceof", - int: "int", - interface: "interface", - let: "let", - long: "long", - mod: "mod", - native: "native", - new: "new", - null: "null", - package: "package", - private: "private", - protected: "protected", - public: "public", - return: "return", - set: "set", - short: "short", - static: "static", - super: "super", - switch: "switch", - synchronized: "synchronized", - this: "this", - throw: "throw", - throws: "throws", - transient: "transient", - true: "true", - try: "try", - typeof: "typeof", - var: "var", - void: "void", - volatile: "volatile", - while: "while", - with: "with", - yield: "yield" -} diff --git a/tests/testdata/npm/run_existing_npm_package/main.out b/tests/testdata/npm/run_existing_npm_package/main.out index 147ed6a995b6b6..373059db5375bf 100644 --- a/tests/testdata/npm/run_existing_npm_package/main.out +++ b/tests/testdata/npm/run_existing_npm_package/main.out @@ -1,3 +1,3 @@ -Download http://localhost:4260/@denotest/bin +Download http://localhost:4260/@denotest%2fbin Download http://localhost:4260/@denotest/bin/0.5.0.tgz Initialize @denotest/bin@0.5.0 diff --git a/tests/testdata/npm/run_existing_npm_package_with_subpath/main.out b/tests/testdata/npm/run_existing_npm_package_with_subpath/main.out deleted file mode 100644 index f6ee03ef1b4adc..00000000000000 --- a/tests/testdata/npm/run_existing_npm_package_with_subpath/main.out +++ /dev/null @@ -1,5 +0,0 @@ -Download http://localhost:4260/@denotest/bin -Download http://localhost:4260/@denotest/bin/1.0.0.tgz -Initialize @denotest/bin@1.0.0 -dev ---help diff --git a/tests/testdata/npm/types_entry_value_not_exists/main.out b/tests/testdata/npm/types_entry_value_not_exists/main.out deleted file mode 100644 index 017a17ea2e74e9..00000000000000 --- a/tests/testdata/npm/types_entry_value_not_exists/main.out +++ /dev/null @@ -1,7 +0,0 @@ -Download http://localhost:4260/@denotest/types-entry-value-not-exists -Download http://localhost:4260/@denotest/types-entry-value-not-exists/1.0.0.tgz -Check file://[WILDCARD]/types_entry_value_not_exists/main.ts -error: TS2322 [ERROR]: Type 'number' is not assignable to type 'string'. -const result: string = getValue(); - ~~~~~~ - at file:///[WILDCARD]/main.ts:4:7 diff --git a/tests/testdata/npm/types_exports_import_types/main.out b/tests/testdata/npm/types_exports_import_types/main.out deleted file mode 100644 index 10f9425ca41e49..00000000000000 --- a/tests/testdata/npm/types_exports_import_types/main.out +++ /dev/null @@ -1,7 +0,0 @@ -Download http://localhost:4260/@denotest/types-exports-import-types -Download http://localhost:4260/@denotest/types-exports-import-types/1.0.0.tgz -Check file://[WILDCARD]/types_exports_import_types/main.ts -error: TS2322 [ERROR]: Type 'number' is not assignable to type 'string'. -const result: string = getValue(); - ~~~~~~ - at file:///[WILDCARD]/main.ts:4:7 diff --git a/tests/testdata/npm/types_no_types_entry/main.out b/tests/testdata/npm/types_no_types_entry/main.out deleted file mode 100644 index 53e872eaf1567a..00000000000000 --- a/tests/testdata/npm/types_no_types_entry/main.out +++ /dev/null @@ -1,13 +0,0 @@ -[UNORDERED_START] -Download http://localhost:4260/@denotest/types-no-types-entry -Download http://localhost:4260/@denotest/types-entry-value-not-exists -[UNORDERED_END] -[UNORDERED_START] -Download http://localhost:4260/@denotest/types-entry-value-not-exists/1.0.0.tgz -Download http://localhost:4260/@denotest/types-no-types-entry/1.0.0.tgz -[UNORDERED_END] -Check file://[WILDCARD]/types_no_types_entry/main.ts -error: TS2322 [ERROR]: Type 'number' is not assignable to type 'string'. -const result: string = getValue(); - ~~~~~~ - at file:///[WILDCARD]/main.ts:4:7 diff --git a/tests/testdata/npm/typescript_file_in_package/main.out b/tests/testdata/npm/typescript_file_in_package/main.out deleted file mode 100644 index 3c3b97190728d5..00000000000000 --- a/tests/testdata/npm/typescript_file_in_package/main.out +++ /dev/null @@ -1,6 +0,0 @@ -Download http://localhost:4260/@denotest/typescript-file -Download http://localhost:4260/@denotest/typescript-file/1.0.0.tgz -error: Could not resolve 'npm:@denotest/typescript-file@1.0.0'. - -Caused by: - TypeScript files are not supported in npm packages: file:///[WILDCARD]/@denotest/typescript-file/1.0.0/index.ts diff --git a/tests/testdata/run/005_more_imports.ts b/tests/testdata/run/005_more_imports.ts deleted file mode 100644 index 6c96fac64c19e9..00000000000000 --- a/tests/testdata/run/005_more_imports.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { printHello3, returnsFoo2, returnsHi } from "../subdir/mod1.ts"; - -printHello3(); - -if (returnsHi() !== "Hi") { - throw Error("Unexpected"); -} - -if (returnsFoo2() !== "Foo") { - throw Error("Unexpected"); -} diff --git a/tests/testdata/run/014_duplicate_import.ts b/tests/testdata/run/014_duplicate_import.ts deleted file mode 100644 index c7dd881cf04d69..00000000000000 --- a/tests/testdata/run/014_duplicate_import.ts +++ /dev/null @@ -1,9 +0,0 @@ -// with all the imports of the same module, the module should only be -// instantiated once -import "../subdir/auto_print_hello.ts"; - -import "../subdir/auto_print_hello.ts"; - -(async () => { - await import("../subdir/auto_print_hello.ts"); -})(); diff --git a/tests/testdata/run/020_json_modules.ts b/tests/testdata/run/020_json_modules.ts deleted file mode 100644 index b4ae6066547f62..00000000000000 --- a/tests/testdata/run/020_json_modules.ts +++ /dev/null @@ -1,2 +0,0 @@ -import config from "../subdir/config.json"; -console.log(JSON.stringify(config)); diff --git a/tests/testdata/run/021_mjs_modules.ts b/tests/testdata/run/021_mjs_modules.ts deleted file mode 100644 index 838cd2c382c6e7..00000000000000 --- a/tests/testdata/run/021_mjs_modules.ts +++ /dev/null @@ -1,2 +0,0 @@ -import { isMod5 } from "../subdir/mod5.mjs"; -console.log(isMod5); diff --git a/tests/testdata/run/035_cached_only_flag.out b/tests/testdata/run/035_cached_only_flag.out deleted file mode 100644 index f677ec915347d6..00000000000000 --- a/tests/testdata/run/035_cached_only_flag.out +++ /dev/null @@ -1 +0,0 @@ -error: Specifier not found in cache: "http://127.0.0.1:4545/run/019_media_types.ts", --cached-only is specified. diff --git a/tests/testdata/run/042_dyn_import_evalcontext.ts b/tests/testdata/run/042_dyn_import_evalcontext.ts deleted file mode 100644 index 386ae48ec7d09b..00000000000000 --- a/tests/testdata/run/042_dyn_import_evalcontext.ts +++ /dev/null @@ -1,5 +0,0 @@ -// @ts-expect-error "Deno[Deno.internal].core" is not a public interface -Deno[Deno.internal].core.evalContext( - "(async () => console.log(await import('./subdir/mod4.js')))()", - new URL("..", import.meta.url).href, -); diff --git a/tests/testdata/run/044_bad_resource.ts b/tests/testdata/run/044_bad_resource.ts deleted file mode 100644 index b956a3e3f2417e..00000000000000 --- a/tests/testdata/run/044_bad_resource.ts +++ /dev/null @@ -1,3 +0,0 @@ -const file = await Deno.open("./run/044_bad_resource.ts", { read: true }); -file.close(); -await file.seek(10, 0); diff --git a/tests/testdata/run/052_no_remote_flag.out b/tests/testdata/run/052_no_remote_flag.out deleted file mode 100644 index 2f5950ee89c66c..00000000000000 --- a/tests/testdata/run/052_no_remote_flag.out +++ /dev/null @@ -1 +0,0 @@ -error: A remote specifier was requested: "http://127.0.0.1:4545/run/019_media_types.ts", but --no-remote is specified. diff --git a/tests/testdata/run/add.cjs b/tests/testdata/run/add.cjs new file mode 100644 index 00000000000000..2a886fbc18bcbc --- /dev/null +++ b/tests/testdata/run/add.cjs @@ -0,0 +1,3 @@ +module.exports.add = function (a, b) { + return a + b; +}; diff --git a/tests/testdata/run/config_json_import.ts b/tests/testdata/run/config_json_import.ts deleted file mode 100644 index 7141f14950dac6..00000000000000 --- a/tests/testdata/run/config_json_import.ts +++ /dev/null @@ -1,2 +0,0 @@ -import config from "../jsx/deno-jsx.json" with { type: "json" }; -console.log(config); diff --git a/tests/testdata/run/env_file_missing.out b/tests/testdata/run/env_file_missing.out deleted file mode 100644 index f50c1789ee48e9..00000000000000 --- a/tests/testdata/run/env_file_missing.out +++ /dev/null @@ -1,4 +0,0 @@ -Warning The `--env-file` flag was used, but the environment file specified 'missing' was not found. -undefined -undefined -undefined diff --git a/tests/testdata/run/error_002.ts b/tests/testdata/run/error_002.ts deleted file mode 100644 index 5f8179bbeb3df6..00000000000000 --- a/tests/testdata/run/error_002.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { throwsError } from "../subdir/mod1.ts"; - -function foo() { - throwsError(); -} - -foo(); diff --git a/tests/testdata/run/error_015_dynamic_import_permissions.js b/tests/testdata/run/error_015_dynamic_import_permissions.js deleted file mode 100644 index 47961cf63b8790..00000000000000 --- a/tests/testdata/run/error_015_dynamic_import_permissions.js +++ /dev/null @@ -1,3 +0,0 @@ -(async () => { - await import("" + "http://localhost:4545/subdir/mod4.js"); -})(); diff --git a/tests/testdata/run/error_015_dynamic_import_permissions.out b/tests/testdata/run/error_015_dynamic_import_permissions.out deleted file mode 100644 index 87ce43e9cd013c..00000000000000 --- a/tests/testdata/run/error_015_dynamic_import_permissions.out +++ /dev/null @@ -1,4 +0,0 @@ -error: Uncaught (in promise) TypeError: Requires net access to "localhost:4545", run again with the --allow-net flag - await import("" + "http://localhost:4545/subdir/mod4.js"); - ^ - at async file://[WILDCARD]/error_015_dynamic_import_permissions.js:2:3 diff --git a/tests/testdata/run/error_no_check.ts b/tests/testdata/run/error_no_check.ts deleted file mode 100644 index 2da01e639a8111..00000000000000 --- a/tests/testdata/run/error_no_check.ts +++ /dev/null @@ -1 +0,0 @@ -export { AnInterface, isAnInterface } from "../subdir/type_and_code.ts"; diff --git a/tests/testdata/run/error_no_check.ts.out b/tests/testdata/run/error_no_check.ts.out deleted file mode 100644 index 78f478045dc2ba..00000000000000 --- a/tests/testdata/run/error_no_check.ts.out +++ /dev/null @@ -1,2 +0,0 @@ -error: Uncaught SyntaxError: The requested module '../subdir/type_and_code.ts' does not provide an export named 'AnInterface' -[WILDCARD] \ No newline at end of file diff --git a/tests/testdata/run/error_type_definitions.ts b/tests/testdata/run/error_type_definitions.ts deleted file mode 100644 index 86675cbaaebeb9..00000000000000 --- a/tests/testdata/run/error_type_definitions.ts +++ /dev/null @@ -1,5 +0,0 @@ -// @deno-types="../type_definitions/bar.d.ts" -import { Bar } from "../type_definitions/bar.js"; - -const bar = new Bar(); -console.log(bar); diff --git a/tests/testdata/run/extension_import.ts.out b/tests/testdata/run/extension_import.ts.out index 88039a9ce8c104..0ff656a9c8549d 100644 --- a/tests/testdata/run/extension_import.ts.out +++ b/tests/testdata/run/extension_import.ts.out @@ -1,8 +1,9 @@ -error: Unsupported scheme "ext" for module "ext:runtime/01_errors.js". Supported schemes: [ - "data", - "blob", - "file", - "http", - "https", -] +error: Unsupported scheme "ext" for module "ext:runtime/01_errors.js". Supported schemes: + - "blob" + - "data" + - "file" + - "http" + - "https" + - "jsr" + - "npm" at [WILDCARD]/extension_import.ts:1:8 diff --git a/tests/testdata/run/fix_dynamic_import_errors.js b/tests/testdata/run/fix_dynamic_import_errors.js deleted file mode 100644 index 1d7be37e06187e..00000000000000 --- a/tests/testdata/run/fix_dynamic_import_errors.js +++ /dev/null @@ -1,7 +0,0 @@ -import("../dynamic_import/b.js").catch(() => { - console.log("caught import error from b.js"); -}); - -import("../dynamic_import/c.js").catch(() => { - console.log("caught import error from c.js"); -}); diff --git a/tests/testdata/run/fix_emittable_skipped.js b/tests/testdata/run/fix_emittable_skipped.js deleted file mode 100644 index a4ccc9efda7f35..00000000000000 --- a/tests/testdata/run/fix_emittable_skipped.js +++ /dev/null @@ -1,7 +0,0 @@ -/// - -import "../subdir/polyfill.ts"; - -export const a = "a"; - -console.log(globalThis.polyfill); diff --git a/tests/testdata/run/fix_js_import_js.ts b/tests/testdata/run/fix_js_import_js.ts deleted file mode 100644 index 0f01877cd81946..00000000000000 --- a/tests/testdata/run/fix_js_import_js.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { isMod4 } from "../subdir/mod6.js"; - -console.log(isMod4); diff --git a/tests/testdata/run/fix_js_imports.ts b/tests/testdata/run/fix_js_imports.ts deleted file mode 100644 index 6ed13bae30c3ed..00000000000000 --- a/tests/testdata/run/fix_js_imports.ts +++ /dev/null @@ -1,3 +0,0 @@ -import * as amdLike from "../subdir/amd_like.js"; - -console.log(amdLike); diff --git a/tests/testdata/run/import_type.ts b/tests/testdata/run/import_type.ts deleted file mode 100644 index 22c639cbcfc7ed..00000000000000 --- a/tests/testdata/run/import_type.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { type B, create } from "../subdir/export_types.ts"; - -const b: B = create(); - -console.log(b); diff --git a/tests/testdata/run/issue13562.ts b/tests/testdata/run/issue13562.ts deleted file mode 100644 index afbf69f99e5b25..00000000000000 --- a/tests/testdata/run/issue13562.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { printHello3 } from "../subdir/mod1.ts?q=.json"; - -printHello3(); diff --git a/tests/testdata/run/jsx_precompile/no_pragma.out b/tests/testdata/run/jsx_precompile/no_pragma.out deleted file mode 100644 index 43799592304e79..00000000000000 --- a/tests/testdata/run/jsx_precompile/no_pragma.out +++ /dev/null @@ -1,3 +0,0 @@ -Download http://localhost:4545/jsx/jsx-precompile/index.ts -Check file:///[WILDCARD]/run/jsx_precompile/no_pragma.tsx -imported http://localhost:4545/jsx/jsx-precompile/index.ts diff --git a/tests/testdata/run/jsx_precompile/skip.out b/tests/testdata/run/jsx_precompile/skip.out deleted file mode 100644 index a32b616f06f186..00000000000000 --- a/tests/testdata/run/jsx_precompile/skip.out +++ /dev/null @@ -1,3 +0,0 @@ -Download http://localhost:4545/jsx/jsx-precompile/index.ts -Check file:///[WILDCARD]/run/jsx_precompile/skip.tsx -imported http://localhost:4545/jsx/jsx-precompile/index.ts diff --git a/tests/testdata/run/lock_check_ok.json b/tests/testdata/run/lock_check_ok.json deleted file mode 100644 index 94de0f630798e9..00000000000000 --- a/tests/testdata/run/lock_check_ok.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "http://127.0.0.1:4545/subdir/print_hello.ts": "fa6692c8f9ff3fb107e773c3ece5274e9d08be282867a1e3ded1d9c00fcaa63c", - "http://127.0.0.1:4545/run/003_relative_import.ts": "a1572e8fd2c2712b33f04aed2561505b5feb2c8696f1f2cded3de7127931b97e" -} diff --git a/tests/testdata/run/lock_v2_check_ok.json b/tests/testdata/run/lock_v2_check_ok.json deleted file mode 100644 index 63bec862a72a10..00000000000000 --- a/tests/testdata/run/lock_v2_check_ok.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "version": "2", - "remote": { - "http://127.0.0.1:4545/subdir/print_hello.ts": "fa6692c8f9ff3fb107e773c3ece5274e9d08be282867a1e3ded1d9c00fcaa63c", - "http://127.0.0.1:4545/run/003_relative_import.ts": "a1572e8fd2c2712b33f04aed2561505b5feb2c8696f1f2cded3de7127931b97e" - } -} diff --git a/tests/testdata/run/lock_write_fetch/main.ts b/tests/testdata/run/lock_write_fetch/main.ts index 4ce63131192537..d8a50a9aa6e7f9 100644 --- a/tests/testdata/run/lock_write_fetch/main.ts +++ b/tests/testdata/run/lock_write_fetch/main.ts @@ -9,6 +9,7 @@ const fetchProc = await new Deno.Command(Deno.execPath(), { stderr: "null", args: [ "cache", + "--allow-import", "--reload", "--lock=lock_write_fetch.json", "--cert=tls/RootCA.pem", @@ -23,6 +24,7 @@ const fetchCheckProc = await new Deno.Command(Deno.execPath(), { stderr: "null", args: [ "cache", + "--allow-import", "--lock=lock_write_fetch.json", "--cert=tls/RootCA.pem", "run/https_import.ts", @@ -38,6 +40,7 @@ const runProc = await new Deno.Command(Deno.execPath(), { stderr: "null", args: [ "run", + "--allow-import", "--lock=lock_write_fetch.json", "--allow-read", "--cert=tls/RootCA.pem", diff --git a/tests/testdata/run/no_check_remote.ts.disabled.out b/tests/testdata/run/no_check_remote.ts.disabled.out deleted file mode 100644 index 3442646348e6ec..00000000000000 --- a/tests/testdata/run/no_check_remote.ts.disabled.out +++ /dev/null @@ -1,4 +0,0 @@ -error: TS2322 [ERROR]: Type '12' is not assignable to type '"a"'. -export const a: "a" = 12; - ^ - at http://localhost:4545/subdir/type_error.ts:1:14 diff --git a/tests/testdata/run/permission_request_no_prompt.ts b/tests/testdata/run/permission_request_no_prompt.ts new file mode 100644 index 00000000000000..e33ffe3ce8c486 --- /dev/null +++ b/tests/testdata/run/permission_request_no_prompt.ts @@ -0,0 +1 @@ +console.log(await Deno.permissions.request({ name: "read" })); diff --git a/tests/testdata/run/permissions_trace.ts b/tests/testdata/run/permissions_trace.ts new file mode 100644 index 00000000000000..d061ac6bf86280 --- /dev/null +++ b/tests/testdata/run/permissions_trace.ts @@ -0,0 +1,9 @@ +function foo() { + Deno.hostname(); +} + +function bar() { + foo(); +} + +bar(); diff --git a/tests/testdata/run/top_level_await/loop.js b/tests/testdata/run/top_level_await/loop.js deleted file mode 100644 index 415db5ec781111..00000000000000 --- a/tests/testdata/run/top_level_await/loop.js +++ /dev/null @@ -1,20 +0,0 @@ -const importsDir = Deno.readDirSync( - Deno.realPathSync("./run/top_level_await/tla2"), -); - -const resolvedPaths = []; - -for (const { name } of importsDir) { - const filePath = Deno.realPathSync(`./run/top_level_await/tla2/${name}`); - resolvedPaths.push(filePath); -} - -resolvedPaths.sort(); - -for (const filePath of resolvedPaths) { - console.log("loading", filePath); - const mod = await import(`file://${filePath}`); - console.log("loaded", mod); -} - -console.log("all loaded"); diff --git a/tests/testdata/run/top_level_await/top_level_await.js b/tests/testdata/run/top_level_await/top_level_await.js deleted file mode 100644 index ea319ea124bd6f..00000000000000 --- a/tests/testdata/run/top_level_await/top_level_await.js +++ /dev/null @@ -1,3 +0,0 @@ -const buf = await Deno.readFile("./assets/hello.txt"); -const n = await Deno.stdout.write(buf); -console.log(`\n\nwrite ${n}`); diff --git a/tests/testdata/run/top_level_await/top_level_await.out b/tests/testdata/run/top_level_await/top_level_await.out deleted file mode 100644 index 4b65d15fe3c0d6..00000000000000 --- a/tests/testdata/run/top_level_await/top_level_await.out +++ /dev/null @@ -1,3 +0,0 @@ -Hello world! - -write 12 diff --git a/tests/testdata/run/top_level_await/top_level_await.ts b/tests/testdata/run/top_level_await/top_level_await.ts deleted file mode 100644 index 8d47ceb21e1cf8..00000000000000 --- a/tests/testdata/run/top_level_await/top_level_await.ts +++ /dev/null @@ -1,3 +0,0 @@ -const buf: Uint8Array = await Deno.readFile("./assets/hello.txt"); -const n: number = await Deno.stdout.write(buf); -console.log(`\n\nwrite ${n}`); diff --git a/tests/testdata/run/type_definitions.ts b/tests/testdata/run/type_definitions.ts deleted file mode 100644 index 5948427219a639..00000000000000 --- a/tests/testdata/run/type_definitions.ts +++ /dev/null @@ -1,12 +0,0 @@ -// deno-lint-ignore-file - -// @deno-types="../type_definitions/foo.d.ts" -import { foo } from "../type_definitions/foo.js"; -// @deno-types="../type_definitions/fizz.d.ts" -import "../type_definitions/fizz.js"; - -import * as qat from "../type_definitions/qat.ts"; - -console.log(foo); -console.log(fizz); -console.log(qat.qat); diff --git a/tests/testdata/run/type_directives_01.ts b/tests/testdata/run/type_directives_01.ts deleted file mode 100644 index 71305824c70120..00000000000000 --- a/tests/testdata/run/type_directives_01.ts +++ /dev/null @@ -1,3 +0,0 @@ -import * as foo from "http://127.0.0.1:4545/xTypeScriptTypes.js"; - -console.log(foo.foo); diff --git a/tests/testdata/run/type_directives_01.ts.out b/tests/testdata/run/type_directives_01.ts.out deleted file mode 100644 index 77ed3ae264b430..00000000000000 --- a/tests/testdata/run/type_directives_01.ts.out +++ /dev/null @@ -1,3 +0,0 @@ -[WILDCARD] -DEBUG TS - host.getSourceFile("http://127.0.0.1:4545/xTypeScriptTypes.d.ts", Latest) -[WILDCARD] \ No newline at end of file diff --git a/tests/testdata/run/type_directives_02.ts b/tests/testdata/run/type_directives_02.ts deleted file mode 100644 index 0c59346e2b0c68..00000000000000 --- a/tests/testdata/run/type_directives_02.ts +++ /dev/null @@ -1,3 +0,0 @@ -import * as foo from "../subdir/type_reference.js"; - -console.log(foo.foo); diff --git a/tests/testdata/run/type_directives_02.ts.out b/tests/testdata/run/type_directives_02.ts.out deleted file mode 100644 index b064483b412437..00000000000000 --- a/tests/testdata/run/type_directives_02.ts.out +++ /dev/null @@ -1,3 +0,0 @@ -[WILDCARD] -DEBUG TS - host.getSourceFile("file:///[WILDCARD]/subdir/type_reference.d.ts", Latest) -[WILDCARD] \ No newline at end of file diff --git a/tests/testdata/run/unhandled_rejection.ts.out b/tests/testdata/run/unhandled_rejection.ts.out deleted file mode 100644 index 6ab1667dc190c1..00000000000000 --- a/tests/testdata/run/unhandled_rejection.ts.out +++ /dev/null @@ -1,9 +0,0 @@ -[WILDCARD] -unhandled rejection at: Promise { - Error: bar not available - at new Foo (file:///[WILDCARD]/testdata/run/unhandled_rejection.ts:8:20) - at file:///[WILDCARD]/testdata/run/unhandled_rejection.ts:12:1 -} reason: Error: bar not available - at new Foo (file:///[WILDCARD]/testdata/run/unhandled_rejection.ts:8:20) - at file:///[WILDCARD]/testdata/run/unhandled_rejection.ts:12:1 -unhandled rejection at: Promise { undefined } reason: undefined diff --git a/tests/testdata/run/unhandled_rejection_sync_error.ts.out b/tests/testdata/run/unhandled_rejection_sync_error.ts.out deleted file mode 100644 index e178373f03b562..00000000000000 --- a/tests/testdata/run/unhandled_rejection_sync_error.ts.out +++ /dev/null @@ -1,6 +0,0 @@ -[WILDCARD] -unhandled rejection at: Promise { - Error: boom! - at file:///[WILDCARD]testdata/run/unhandled_rejection_sync_error.ts:6:7 -} reason: Error: boom! - at file:///[WILDCARD]testdata/run/unhandled_rejection_sync_error.ts:6:7 diff --git a/tests/testdata/run/unstable_worker.ts b/tests/testdata/run/unstable_worker.ts deleted file mode 100644 index d111d2c7e3dbed..00000000000000 --- a/tests/testdata/run/unstable_worker.ts +++ /dev/null @@ -1,6 +0,0 @@ -const w = new Worker( - import.meta.resolve("../workers/worker_unstable.ts"), - { type: "module", name: "Unstable Worker" }, -); - -w.postMessage({}); diff --git a/tests/testdata/run/unsupported_dynamic_import_scheme.out b/tests/testdata/run/unsupported_dynamic_import_scheme.out deleted file mode 100644 index c708fced493a41..00000000000000 --- a/tests/testdata/run/unsupported_dynamic_import_scheme.out +++ /dev/null @@ -1,7 +0,0 @@ -error: Uncaught (in promise) TypeError: Unsupported scheme "xxx" for module "xxx:". Supported schemes: [ - "data", - "blob", - "file", - "http", - "https", -] diff --git a/tests/testdata/run/warn_on_deprecated_api/main.js b/tests/testdata/run/warn_on_deprecated_api/main.js deleted file mode 100644 index 8811df78de2792..00000000000000 --- a/tests/testdata/run/warn_on_deprecated_api/main.js +++ /dev/null @@ -1,34 +0,0 @@ -import { runEcho as runEcho2 } from "http://localhost:4545/run/warn_on_deprecated_api/mod.ts"; - -// @ts-ignore `Deno.run()` was soft-removed in Deno 2. -const p = Deno.run({ - cmd: [ - Deno.execPath(), - "eval", - "console.log('hello world')", - ], -}); -await p.status(); -p.close(); - -async function runEcho() { - // @ts-ignore `Deno.run()` was soft-removed in Deno 2. - const p = Deno.run({ - cmd: [ - Deno.execPath(), - "eval", - "console.log('hello world')", - ], - }); - await p.status(); - p.close(); -} - -await runEcho(); -await runEcho(); - -for (let i = 0; i < 10; i++) { - await runEcho(); -} - -await runEcho2(); diff --git a/tests/testdata/run/warn_on_deprecated_api/main.out b/tests/testdata/run/warn_on_deprecated_api/main.out deleted file mode 100644 index ef85a6f99bac3f..00000000000000 --- a/tests/testdata/run/warn_on_deprecated_api/main.out +++ /dev/null @@ -1,15 +0,0 @@ -Download http://localhost:4545/run/warn_on_deprecated_api/mod.ts -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world diff --git a/tests/testdata/run/warn_on_deprecated_api/main.verbose.out b/tests/testdata/run/warn_on_deprecated_api/main.verbose.out deleted file mode 100644 index e17562eef31a8e..00000000000000 --- a/tests/testdata/run/warn_on_deprecated_api/main.verbose.out +++ /dev/null @@ -1,60 +0,0 @@ -Download http://localhost:4545/run/warn_on_deprecated_api/mod.ts - -See the Deno 1 to 2 Migration Guide for more information at https://docs.deno.com/runtime/manual/advanced/migrate_deprecations - -Stack trace: - at [WILDCARD]warn_on_deprecated_api/main.js:3:16 - -hint: Use "Deno.Command()" API instead. - -hello world - -See the Deno 1 to 2 Migration Guide for more information at https://docs.deno.com/runtime/manual/advanced/migrate_deprecations - -Stack trace: - at runEcho ([WILDCARD]warn_on_deprecated_api/main.js:14:18) - at [WILDCARD]warn_on_deprecated_api/main.js:25:7 - -hint: Use "Deno.Command()" API instead. - -hello world - -See the Deno 1 to 2 Migration Guide for more information at https://docs.deno.com/runtime/manual/advanced/migrate_deprecations - -Stack trace: - at runEcho ([WILDCARD]warn_on_deprecated_api/main.js:14:18) - at [WILDCARD]warn_on_deprecated_api/main.js:26:7 - -hint: Use "Deno.Command()" API instead. - -hello world - -See the Deno 1 to 2 Migration Guide for more information at https://docs.deno.com/runtime/manual/advanced/migrate_deprecations - -Stack trace: - at runEcho ([WILDCARD]warn_on_deprecated_api/main.js:14:18) - at [WILDCARD]warn_on_deprecated_api/main.js:29:9 - -hint: Use "Deno.Command()" API instead. - -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world - -See the Deno 1 to 2 Migration Guide for more information at https://docs.deno.com/runtime/manual/advanced/migrate_deprecations - -Stack trace: - at runEcho (http://localhost:4545/run/warn_on_deprecated_api/mod.ts:2:18) - at [WILDCARD]warn_on_deprecated_api/main.js:32:7 - -hint: Use "Deno.Command()" API instead. -hint: It appears this API is used by a remote dependency. Try upgrading to the latest version of that dependency. - -hello world diff --git a/tests/testdata/run/warn_on_deprecated_api/main_disabled_env.out b/tests/testdata/run/warn_on_deprecated_api/main_disabled_env.out deleted file mode 100644 index ef85a6f99bac3f..00000000000000 --- a/tests/testdata/run/warn_on_deprecated_api/main_disabled_env.out +++ /dev/null @@ -1,15 +0,0 @@ -Download http://localhost:4545/run/warn_on_deprecated_api/mod.ts -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world diff --git a/tests/testdata/run/warn_on_deprecated_api/main_disabled_flag.out b/tests/testdata/run/warn_on_deprecated_api/main_disabled_flag.out deleted file mode 100644 index ce3755d16683cc..00000000000000 --- a/tests/testdata/run/warn_on_deprecated_api/main_disabled_flag.out +++ /dev/null @@ -1,14 +0,0 @@ -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world -hello world diff --git a/tests/testdata/run/warn_on_deprecated_api/mod.ts b/tests/testdata/run/warn_on_deprecated_api/mod.ts deleted file mode 100644 index f74632b2cdfed0..00000000000000 --- a/tests/testdata/run/warn_on_deprecated_api/mod.ts +++ /dev/null @@ -1,11 +0,0 @@ -export async function runEcho() { - const p = Deno.run({ - cmd: [ - Deno.execPath(), - "eval", - "console.log('hello world')", - ], - }); - await p.status(); - p.close(); -} diff --git a/tests/testdata/run/with_package_json/with_stop/main.out b/tests/testdata/run/with_package_json/with_stop/main.out deleted file mode 100644 index 44098a2d863c5c..00000000000000 --- a/tests/testdata/run/with_package_json/with_stop/main.out +++ /dev/null @@ -1,5 +0,0 @@ -[WILDCARD]Config file found at '[WILDCARD]with_package_json[WILDCARD]with_stop[WILDCARD]some[WILDCARD]nested[WILDCARD]deno.json' -[WILDCARD] -error: Relative import path "chalk" not prefixed with / or ./ or ../ - hint: If you want to use a JSR or npm package, try running `deno add jsr:chalk` or `deno add npm:chalk` - at file:///[WILDCARD]with_package_json/with_stop/some/nested/dir/main.ts:3:19 diff --git a/tests/testdata/run/worker_close_in_wasm_reactions.js b/tests/testdata/run/worker_close_in_wasm_reactions.js deleted file mode 100644 index 95f34e944db1c1..00000000000000 --- a/tests/testdata/run/worker_close_in_wasm_reactions.js +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -// https://github.com/denoland/deno/issues/12263 -// Test for a panic that happens when a worker is closed in the reactions of a -// WASM async operation. - -new Worker( - import.meta.resolve("../workers/close_in_wasm_reactions.js"), - { type: "module" }, -); diff --git a/tests/testdata/run/worker_event_handler_test.js b/tests/testdata/run/worker_event_handler_test.js deleted file mode 100644 index a91b0ec0b161d1..00000000000000 --- a/tests/testdata/run/worker_event_handler_test.js +++ /dev/null @@ -1,5 +0,0 @@ -const w = new Worker( - import.meta.resolve("../workers/worker_event_handlers.js"), - { type: "module" }, -); -w.postMessage({}); diff --git a/tests/testdata/task/package_json_node_modules_dir_false/bin.out b/tests/testdata/task/package_json_node_modules_dir_false/bin.out deleted file mode 100644 index d5d59d551509e1..00000000000000 --- a/tests/testdata/task/package_json_node_modules_dir_false/bin.out +++ /dev/null @@ -1,2 +0,0 @@ -Task echo deno eval 'console.log(1)' -1 diff --git a/tests/testdata/task/package_json_node_modules_dir_false/deno.json b/tests/testdata/task/package_json_node_modules_dir_false/deno.json deleted file mode 100644 index 23a325cfc23fbd..00000000000000 --- a/tests/testdata/task/package_json_node_modules_dir_false/deno.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "nodeModulesDir": false -} diff --git a/tests/testdata/task/package_json_node_modules_dir_false/package.json b/tests/testdata/task/package_json_node_modules_dir_false/package.json deleted file mode 100644 index 081e076b9f8874..00000000000000 --- a/tests/testdata/task/package_json_node_modules_dir_false/package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "scripts": { - "echo": "deno eval 'console.log(1)'" - }, - "dependencies": { - "@denotest/bin": "0.5", - "other": "npm:@denotest/bin@1.0" - } -} diff --git a/tests/testdata/wasm/math.ts b/tests/testdata/wasm/math.ts new file mode 100644 index 00000000000000..5a1de0865e775b --- /dev/null +++ b/tests/testdata/wasm/math.ts @@ -0,0 +1,8 @@ +// this file is imported by math_with_import.wasm +export function add(a: number, b: number) { + return a + b; +} + +export function subtract(a: number, b: number) { + return a - b; +} diff --git a/tests/testdata/wasm/math.wasm b/tests/testdata/wasm/math.wasm new file mode 100644 index 00000000000000..6b3950fcc5297b Binary files /dev/null and b/tests/testdata/wasm/math.wasm differ diff --git a/tests/testdata/wasm/math_with_import.wasm b/tests/testdata/wasm/math_with_import.wasm new file mode 100644 index 00000000000000..64e195e42b6a2e Binary files /dev/null and b/tests/testdata/wasm/math_with_import.wasm differ diff --git a/tests/testdata/workers/close_in_wasm_reactions.js b/tests/testdata/workers/close_in_wasm_reactions.js index abe5731085bff8..2cb0e4a7680217 100644 --- a/tests/testdata/workers/close_in_wasm_reactions.js +++ b/tests/testdata/workers/close_in_wasm_reactions.js @@ -1,6 +1,6 @@ // https://github.com/denoland/deno/issues/12263 // Test for a panic that happens when a worker is closed in the reactions of a -// WASM async operation. +// Wasm async operation. // The minimum valid wasm module, plus two additional zero bytes. const buffer = new Uint8Array([ diff --git a/tests/testdata/workers/permissions_dynamic_remote.ts b/tests/testdata/workers/permissions_dynamic_remote.ts deleted file mode 100644 index 54a361bc0055b7..00000000000000 --- a/tests/testdata/workers/permissions_dynamic_remote.ts +++ /dev/null @@ -1,11 +0,0 @@ -new Worker( - "http://localhost:4545/workers/dynamic_remote.ts", - { - type: "module", - deno: { - permissions: { - net: false, - }, - }, - }, -); diff --git a/tests/testdata/workers/static_remote.ts b/tests/testdata/workers/static_remote.ts index 2d6e820fd687a6..6bb64eccdb14c6 100644 --- a/tests/testdata/workers/static_remote.ts +++ b/tests/testdata/workers/static_remote.ts @@ -1,2 +1,2 @@ -// This file doesn't really exist, but it doesn't matter, a "PermissionsDenied" error should be thrown. +// This file doesn't really exist, but it doesn't matter, a "NotCapable" error should be thrown. import "https://example.com/some/file.ts"; diff --git a/tests/unit/command_test.ts b/tests/unit/command_test.ts index 51bbdd8600f7bd..8345548f8559d0 100644 --- a/tests/unit/command_test.ts +++ b/tests/unit/command_test.ts @@ -14,27 +14,31 @@ Deno.test( const enc = new TextEncoder(); const cwd = await Deno.makeTempDir({ prefix: "deno_command_test" }); + const exitCodeFileLock = "deno_was_here.lock"; const exitCodeFile = "deno_was_here"; const programFile = "poll_exit.ts"; const program = ` +const file = await Deno.open("${exitCodeFileLock}", { write: true, create: true }); async function tryExit() { + await file.lock(true); try { const code = parseInt(await Deno.readTextFile("${exitCodeFile}")); Deno.exit(code); } catch { // Retry if we got here before deno wrote the file. setTimeout(tryExit, 0.01); + } finally { + await file.unlock(); } } tryExit(); `; - Deno.writeFileSync(`${cwd}/${programFile}`, enc.encode(program)); const command = new Deno.Command(Deno.execPath(), { cwd, - args: ["run", "--allow-read", programFile], + args: ["run", "-RW", programFile], stdout: "inherit", stderr: "inherit", }); @@ -43,12 +47,18 @@ tryExit(); // Write the expected exit code *after* starting deno. // This is how we verify that `Child` is actually asynchronous. const code = 84; - Deno.writeFileSync(`${cwd}/${exitCodeFile}`, enc.encode(`${code}`)); + await using file = await Deno.open(`${cwd}/${exitCodeFileLock}`, { + write: true, + create: true, + }); + await file.lock(true); + Deno.writeFileSync(`${cwd}/${exitCodeFile}`, enc.encode(`${code}`)); + await file.unlock(); const status = await child.status; await Deno.remove(cwd, { recursive: true }); - assertEquals(status.success, false); assertEquals(status.code, code); + assertEquals(status.success, false); assertEquals(status.signal, null); }, ); diff --git a/tests/unit/console_test.ts b/tests/unit/console_test.ts index 201d18f0048b75..06f5dd7e6198e5 100644 --- a/tests/unit/console_test.ts +++ b/tests/unit/console_test.ts @@ -1162,7 +1162,7 @@ Deno.test(function consoleTestWithIntegerFormatSpecifier() { assertEquals(stringify("%i"), "%i"); assertEquals(stringify("%i", 42.0), "42"); assertEquals(stringify("%i", 42), "42"); - assertEquals(stringify("%i", "42"), "NaN"); + assertEquals(stringify("%i", "42"), "42"); assertEquals(stringify("%i", 1.5), "1"); assertEquals(stringify("%i", -0.5), "0"); assertEquals(stringify("%i", ""), "NaN"); @@ -1172,7 +1172,7 @@ Deno.test(function consoleTestWithIntegerFormatSpecifier() { assertEquals(stringify("%d", 12345678901234567890123), "1"); assertEquals( stringify("%i", 12345678901234567890123n), - "12345678901234567890123n", + "1.2345678901234568e+22", ); }); @@ -1180,13 +1180,13 @@ Deno.test(function consoleTestWithFloatFormatSpecifier() { assertEquals(stringify("%f"), "%f"); assertEquals(stringify("%f", 42.0), "42"); assertEquals(stringify("%f", 42), "42"); - assertEquals(stringify("%f", "42"), "NaN"); + assertEquals(stringify("%f", "42"), "42"); assertEquals(stringify("%f", 1.5), "1.5"); assertEquals(stringify("%f", -0.5), "-0.5"); assertEquals(stringify("%f", Math.PI), "3.141592653589793"); assertEquals(stringify("%f", ""), "NaN"); assertEquals(stringify("%f", Symbol("foo")), "NaN"); - assertEquals(stringify("%f", 5n), "NaN"); + assertEquals(stringify("%f", 5n), "5"); assertEquals(stringify("%f %f", 42, 43), "42 43"); assertEquals(stringify("%f %f", 42), "42 %f"); }); @@ -1227,6 +1227,7 @@ Deno.test(function consoleParseCssColor() { assertEquals(parseCssColor("inherit"), null); assertEquals(parseCssColor("black"), [0, 0, 0]); assertEquals(parseCssColor("darkmagenta"), [139, 0, 139]); + assertEquals(parseCssColor("darkMaGenta"), [139, 0, 139]); assertEquals(parseCssColor("slateblue"), [106, 90, 205]); assertEquals(parseCssColor("#ffaa00"), [255, 170, 0]); assertEquals(parseCssColor("#ffAA00"), [255, 170, 0]); @@ -1913,6 +1914,21 @@ Deno.test(function consoleLogWhenCauseIsAssignedShouldNotPrintCauseTwice() { }); }); +Deno.test(function consoleLogCauseNotFilteredOnNonError() { + mockConsole((console, out) => { + const foo = { + a: 1, + b: 2, + cause: 3, + }; + console.log(foo); + + const result = stripAnsiCode(out.toString()); + const expected = "{ a: 1, b: 2, cause: 3 }\n"; + assertEquals(result.trim(), expected.trim()); + }); +}); + // console.log(new Proxy(new RegExp(), {})) Deno.test(function consoleLogShouldNotThrowErrorWhenInputIsProxiedRegExp() { mockConsole((console, out) => { diff --git a/tests/unit/fetch_test.ts b/tests/unit/fetch_test.ts index 3ae96746a7d127..298a266903394f 100644 --- a/tests/unit/fetch_test.ts +++ b/tests/unit/fetch_test.ts @@ -439,6 +439,58 @@ Deno.test( }, ); +Deno.test( + { + permissions: { net: true }, + }, + async function fetchWithAuthorizationHeaderRedirection() { + const response = await fetch("http://localhost:4546/echo_server", { + headers: { authorization: "Bearer foo" }, + }); + assertEquals(response.status, 200); + assertEquals(response.statusText, "OK"); + assertEquals(response.url, "http://localhost:4545/echo_server"); + assertEquals(response.headers.get("authorization"), null); + assertEquals(await response.text(), ""); + }, +); + +Deno.test( + { + permissions: { net: true }, + }, + async function fetchWithCookieHeaderRedirection() { + const response = await fetch("http://localhost:4546/echo_server", { + headers: { Cookie: "sessionToken=verySecret" }, + }); + assertEquals(response.status, 200); + assertEquals(response.statusText, "OK"); + assertEquals(response.url, "http://localhost:4545/echo_server"); + assertEquals(response.headers.get("cookie"), null); + assertEquals(await response.text(), ""); + }, +); + +Deno.test( + { + permissions: { net: true }, + }, + async function fetchWithProxyAuthorizationHeaderRedirection() { + const response = await fetch("http://localhost:4546/echo_server", { + headers: { + "proxy-authorization": "Basic ZXNwZW46a29rb3M=", + "accept": "application/json", + }, + }); + assertEquals(response.status, 200); + assertEquals(response.statusText, "OK"); + assertEquals(response.url, "http://localhost:4545/echo_server"); + assertEquals(response.headers.get("proxy-authorization"), null); + assertEquals(response.headers.get("accept"), "application/json"); + assertEquals(await response.text(), ""); + }, +); + Deno.test( { permissions: { net: true } }, async function fetchInitStringBody() { @@ -2119,3 +2171,30 @@ Deno.test( await server; }, ); + +Deno.test("fetch async iterable", async () => { + const iterable = (async function* () { + yield new Uint8Array([1, 2, 3, 4, 5]); + yield new Uint8Array([6, 7, 8, 9, 10]); + })(); + const res = new Response(iterable); + const actual = await res.bytes(); + const expected = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + assertEquals(actual, expected); +}); + +Deno.test("fetch iterable", async () => { + const iterable = (function* () { + yield new Uint8Array([1, 2, 3, 4, 5]); + yield new Uint8Array([6, 7, 8, 9, 10]); + })(); + const res = new Response(iterable); + const actual = await res.bytes(); + const expected = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + assertEquals(actual, expected); +}); + +Deno.test("fetch string object", async () => { + const res = new Response(Object("hello")); + assertEquals(await res.text(), "hello"); +}); diff --git a/tests/unit/fs_events_test.ts b/tests/unit/fs_events_test.ts index cc2f2cd5711242..7489626b9f68bc 100644 --- a/tests/unit/fs_events_test.ts +++ b/tests/unit/fs_events_test.ts @@ -45,6 +45,14 @@ async function makeTempDir(): Promise { return testDir; } +async function makeTempFile(): Promise { + const testFile = await Deno.makeTempFile(); + // The watcher sometimes witnesses the creation of it's own root + // directory. Delay a bit. + await delay(100); + return testFile; +} + Deno.test( { permissions: { read: true, write: true } }, async function watchFsBasic() { @@ -155,3 +163,25 @@ Deno.test( assert(done); }, ); + +Deno.test( + { permissions: { read: true, write: true } }, + async function watchFsRemove() { + const testFile = await makeTempFile(); + using watcher = Deno.watchFs(testFile); + async function waitForRemove() { + for await (const event of watcher) { + if (event.kind === "remove") { + return event; + } + } + } + const eventPromise = waitForRemove(); + + await Deno.remove(testFile); + + // Expect zero events. + const event = await eventPromise; + assertEquals(event!.kind, "remove"); + }, +); diff --git a/tests/unit/globals_test.ts b/tests/unit/globals_test.ts index 45a0458357fcf1..6de228e1c904f2 100644 --- a/tests/unit/globals_test.ts +++ b/tests/unit/globals_test.ts @@ -1,4 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +// deno-lint-ignore-file no-node-globals import { assert, diff --git a/tests/unit/http_test.ts b/tests/unit/http_test.ts index ea72806d3af4f7..355b155afd5afd 100644 --- a/tests/unit/http_test.ts +++ b/tests/unit/http_test.ts @@ -2305,7 +2305,7 @@ Deno.test( const buf = new Uint8Array(128); const readPromise = serverConn.read(buf); - assertThrows(() => Deno.serveHttp(serverConn), Deno.errors.BadResource); + assertThrows(() => Deno.serveHttp(serverConn), Deno.errors.Busy); clientConn.close(); listener.close(); @@ -2338,7 +2338,7 @@ Deno.test( const buf = new Uint8Array(128); const readPromise = serverConn.read(buf); - assertThrows(() => Deno.serveHttp(serverConn), Deno.errors.BadResource); + assertThrows(() => Deno.serveHttp(serverConn), Deno.errors.Busy); clientConn.close(); listener.close(); @@ -2362,7 +2362,7 @@ Deno.test( const buf = new Uint8Array(128); const readPromise = serverConn.read(buf); - assertThrows(() => Deno.serveHttp(serverConn), Deno.errors.BadResource); + assertThrows(() => Deno.serveHttp(serverConn), Deno.errors.Busy); clientConn.close(); listener.close(); diff --git a/tests/unit/net_test.ts b/tests/unit/net_test.ts index c243da47fa0e52..cfa42b3d360135 100644 --- a/tests/unit/net_test.ts +++ b/tests/unit/net_test.ts @@ -126,8 +126,6 @@ Deno.test( const listener = Deno.listen({ port: listenPort }); const p = listener.accept(); listener.close(); - // TODO(piscisaureus): the error type should be `Interrupted` here, which - // gets thrown, but then ext/net catches it and rethrows `BadResource`. await assertRejects( () => p, Deno.errors.BadResource, @@ -168,7 +166,7 @@ Deno.test( } else if (e.message === "Another accept task is ongoing") { acceptErrCount++; } else { - throw new Error("Unexpected error message"); + throw e; } }; const p = listener.accept().catch(checkErr); diff --git a/tests/unit/permissions_test.ts b/tests/unit/permissions_test.ts index 82524d556c301f..f981b10fd6975a 100644 --- a/tests/unit/permissions_test.ts +++ b/tests/unit/permissions_test.ts @@ -20,54 +20,72 @@ Deno.test(function permissionInvalidNameSync() { }, TypeError); }); -Deno.test(async function permissionNetInvalidHost() { - await assertRejects(async () => { - await Deno.permissions.query({ name: "net", host: ":" }); - }, URIError); -}); - -Deno.test(function permissionNetInvalidHostSync() { - assertThrows(() => { - Deno.permissions.querySync({ name: "net", host: ":" }); - }, URIError); -}); - -Deno.test(async function permissionSysValidKind() { - await Deno.permissions.query({ name: "sys", kind: "loadavg" }); - await Deno.permissions.query({ name: "sys", kind: "osRelease" }); - await Deno.permissions.query({ name: "sys", kind: "osUptime" }); - await Deno.permissions.query({ name: "sys", kind: "networkInterfaces" }); - await Deno.permissions.query({ name: "sys", kind: "systemMemoryInfo" }); - await Deno.permissions.query({ name: "sys", kind: "hostname" }); - await Deno.permissions.query({ name: "sys", kind: "uid" }); - await Deno.permissions.query({ name: "sys", kind: "gid" }); - await Deno.permissions.query({ name: "sys", kind: "cpus" }); -}); - -Deno.test(function permissionSysValidKindSync() { - Deno.permissions.querySync({ name: "sys", kind: "loadavg" }); - Deno.permissions.querySync({ name: "sys", kind: "osRelease" }); - Deno.permissions.querySync({ name: "sys", kind: "networkInterfaces" }); - Deno.permissions.querySync({ name: "sys", kind: "systemMemoryInfo" }); - Deno.permissions.querySync({ name: "sys", kind: "hostname" }); - Deno.permissions.querySync({ name: "sys", kind: "uid" }); - Deno.permissions.querySync({ name: "sys", kind: "gid" }); - Deno.permissions.querySync({ name: "sys", kind: "cpus" }); -}); - -Deno.test(async function permissionSysInvalidKind() { - await assertRejects(async () => { - // deno-lint-ignore no-explicit-any - await Deno.permissions.query({ name: "sys", kind: "abc" as any }); - }, TypeError); -}); - -Deno.test(function permissionSysInvalidKindSync() { - assertThrows(() => { - // deno-lint-ignore no-explicit-any - Deno.permissions.querySync({ name: "sys", kind: "abc" as any }); - }, TypeError); -}); +Deno.test( + { permissions: { net: [] } }, + async function permissionNetInvalidHost() { + await assertRejects(async () => { + await Deno.permissions.query({ name: "net", host: ":" }); + }, URIError); + }, +); + +Deno.test( + { permissions: { net: [] } }, + function permissionNetInvalidHostSync() { + assertThrows(() => { + Deno.permissions.querySync({ name: "net", host: ":" }); + }, URIError); + }, +); + +Deno.test( + { permissions: { sys: [] } }, + async function permissionSysValidKind() { + await Deno.permissions.query({ name: "sys", kind: "loadavg" }); + await Deno.permissions.query({ name: "sys", kind: "osRelease" }); + await Deno.permissions.query({ name: "sys", kind: "osUptime" }); + await Deno.permissions.query({ name: "sys", kind: "networkInterfaces" }); + await Deno.permissions.query({ name: "sys", kind: "systemMemoryInfo" }); + await Deno.permissions.query({ name: "sys", kind: "hostname" }); + await Deno.permissions.query({ name: "sys", kind: "uid" }); + await Deno.permissions.query({ name: "sys", kind: "gid" }); + await Deno.permissions.query({ name: "sys", kind: "cpus" }); + }, +); + +Deno.test( + { permissions: { sys: [] } }, + function permissionSysValidKindSync() { + Deno.permissions.querySync({ name: "sys", kind: "loadavg" }); + Deno.permissions.querySync({ name: "sys", kind: "osRelease" }); + Deno.permissions.querySync({ name: "sys", kind: "networkInterfaces" }); + Deno.permissions.querySync({ name: "sys", kind: "systemMemoryInfo" }); + Deno.permissions.querySync({ name: "sys", kind: "hostname" }); + Deno.permissions.querySync({ name: "sys", kind: "uid" }); + Deno.permissions.querySync({ name: "sys", kind: "gid" }); + Deno.permissions.querySync({ name: "sys", kind: "cpus" }); + }, +); + +Deno.test( + { permissions: { sys: [] } }, + async function permissionSysInvalidKind() { + await assertRejects(async () => { + // deno-lint-ignore no-explicit-any + await Deno.permissions.query({ name: "sys", kind: "abc" as any }); + }, TypeError); + }, +); + +Deno.test( + { permissions: { sys: [] } }, + function permissionSysInvalidKindSync() { + assertThrows(() => { + // deno-lint-ignore no-explicit-any + Deno.permissions.querySync({ name: "sys", kind: "abc" as any }); + }, TypeError); + }, +); Deno.test(async function permissionQueryReturnsEventTarget() { const status = await Deno.permissions.query({ name: "read", path: "." }); @@ -134,29 +152,35 @@ Deno.test(function permissionStatusIllegalConstructor() { }); // Regression test for https://github.com/denoland/deno/issues/17020 -Deno.test(async function permissionURL() { - const path = new URL(".", import.meta.url); - - await Deno.permissions.query({ name: "read", path }); - await Deno.permissions.query({ name: "write", path }); - await Deno.permissions.query({ name: "ffi", path }); - await Deno.permissions.query({ name: "run", command: path }); -}); - -Deno.test(function permissionURLSync() { - Deno.permissions.querySync({ - name: "read", - path: new URL(".", import.meta.url), - }); - Deno.permissions.querySync({ - name: "write", - path: new URL(".", import.meta.url), - }); - Deno.permissions.querySync({ - name: "run", - command: new URL(".", import.meta.url), - }); -}); +Deno.test( + { permissions: { read: [], write: [], ffi: [], run: [] } }, + async function permissionURL() { + const path = new URL(".", import.meta.url); + + await Deno.permissions.query({ name: "read", path }); + await Deno.permissions.query({ name: "write", path }); + await Deno.permissions.query({ name: "ffi", path }); + await Deno.permissions.query({ name: "run", command: path }); + }, +); + +Deno.test( + { permissions: { read: [], write: [], ffi: [], run: [] } }, + function permissionURLSync() { + Deno.permissions.querySync({ + name: "read", + path: new URL(".", import.meta.url), + }); + Deno.permissions.querySync({ + name: "write", + path: new URL(".", import.meta.url), + }); + Deno.permissions.querySync({ + name: "run", + command: new URL(".", import.meta.url), + }); + }, +); Deno.test(async function permissionDescriptorValidation() { for (const value of [undefined, null, {}]) { diff --git a/tests/unit/serve_test.ts b/tests/unit/serve_test.ts index 439d71d5533fa0..f5896bc64b26c1 100644 --- a/tests/unit/serve_test.ts +++ b/tests/unit/serve_test.ts @@ -4270,3 +4270,112 @@ Deno.test({ assertEquals(hostname, "0.0.0.0"); await server.shutdown(); }); + +Deno.test({ + name: "AbortSignal aborted when request is cancelled", +}, async () => { + const { promise, resolve } = Promise.withResolvers(); + + let cancelled = false; + + const server = Deno.serve({ + hostname: "0.0.0.0", + port: servePort, + onListen: () => resolve(), + }, async (request) => { + request.signal.addEventListener("abort", () => cancelled = true); + assert(!request.signal.aborted); + await new Promise((resolve) => setTimeout(resolve, 3000)); // abort during waiting + assert(request.signal.aborted); + return new Response("Ok"); + }); + + await promise; + await fetch(`http://localhost:${servePort}/`, { + signal: AbortSignal.timeout(1000), + }).catch(() => {}); + + await server.shutdown(); + + assert(cancelled); +}); + +Deno.test({ + name: "AbortSignal event aborted when request is cancelled", +}, async () => { + const { promise, resolve } = Promise.withResolvers(); + + const server = Deno.serve({ + hostname: "0.0.0.0", + port: servePort, + onListen: () => resolve(), + }, async (request) => { + const { promise: promiseAbort, resolve: resolveAbort } = Promise + .withResolvers(); + request.signal.addEventListener("abort", () => resolveAbort()); + assert(!request.signal.aborted); + + await promiseAbort; + + return new Response("Ok"); + }); + + await promise; + await fetch(`http://localhost:${servePort}/`, { + signal: AbortSignal.timeout(100), + }).catch(() => {}); + + await server.shutdown(); +}); + +// https://github.com/denoland/deno/issues/27083 +Deno.test( + { permissions: { net: true } }, + async function httpServerWebSocketInspectRequest() { + const ac = new AbortController(); + const listeningDeferred = Promise.withResolvers(); + const doneDeferred = Promise.withResolvers(); + const server = Deno.serve({ + handler: (request) => { + const { + response, + socket, + } = Deno.upgradeWebSocket(request); + + socket.onopen = () => { + Deno.inspect(request); // should not throw + }; + socket.onerror = (e) => { + console.error(e); + fail(); + }; + socket.onmessage = (m) => { + socket.send(m.data); + socket.close(1001); + }; + socket.onclose = () => doneDeferred.resolve(); + return response; + }, + port: servePort, + signal: ac.signal, + onListen: onListen(listeningDeferred.resolve), + onError: createOnErrorCb(ac), + }); + + await listeningDeferred.promise; + const def = Promise.withResolvers(); + const ws = new WebSocket(`ws://localhost:${servePort}`); + ws.onmessage = (m) => assertEquals(m.data, "foo"); + ws.onerror = (e) => { + console.error(e); + fail(); + }; + ws.onclose = () => def.resolve(); + ws.onopen = () => ws.send("foo"); + + await def.promise; + await doneDeferred.promise; + ac.abort(); + await server.finished; + }, +); diff --git a/tests/unit/signal_test.ts b/tests/unit/signal_test.ts index 65b5ba78e12b0e..8923aa75bf5441 100644 --- a/tests/unit/signal_test.ts +++ b/tests/unit/signal_test.ts @@ -5,101 +5,101 @@ Deno.test( { ignore: Deno.build.os !== "windows" }, function signalsNotImplemented() { const msg = - "Windows only supports ctrl-c (SIGINT) and ctrl-break (SIGBREAK)."; + "Windows only supports ctrl-c (SIGINT) and ctrl-break (SIGBREAK), but got "; assertThrows( () => { Deno.addSignalListener("SIGALRM", () => {}); }, Error, - msg, + msg + "SIGALRM", ); assertThrows( () => { Deno.addSignalListener("SIGCHLD", () => {}); }, Error, - msg, + msg + "SIGCHLD", ); assertThrows( () => { Deno.addSignalListener("SIGHUP", () => {}); }, Error, - msg, + msg + "SIGHUP", ); assertThrows( () => { Deno.addSignalListener("SIGIO", () => {}); }, Error, - msg, + msg + "SIGIO", ); assertThrows( () => { Deno.addSignalListener("SIGPIPE", () => {}); }, Error, - msg, + msg + "SIGPIPE", ); assertThrows( () => { Deno.addSignalListener("SIGQUIT", () => {}); }, Error, - msg, + msg + "SIGQUIT", ); assertThrows( () => { Deno.addSignalListener("SIGTERM", () => {}); }, Error, - msg, + msg + "SIGTERM", ); assertThrows( () => { Deno.addSignalListener("SIGUSR1", () => {}); }, Error, - msg, + msg + "SIGUSR1", ); assertThrows( () => { Deno.addSignalListener("SIGUSR2", () => {}); }, Error, - msg, + msg + "SIGUSR2", ); assertThrows( () => { Deno.addSignalListener("SIGWINCH", () => {}); }, Error, - msg, + msg + "SIGWINCH", ); assertThrows( () => Deno.addSignalListener("SIGKILL", () => {}), Error, - msg, + msg + "SIGKILL", ); assertThrows( () => Deno.addSignalListener("SIGSTOP", () => {}), Error, - msg, + msg + "SIGSTOP", ); assertThrows( () => Deno.addSignalListener("SIGILL", () => {}), Error, - msg, + msg + "SIGILL", ); assertThrows( () => Deno.addSignalListener("SIGFPE", () => {}), Error, - msg, + msg + "SIGFPE", ); assertThrows( () => Deno.addSignalListener("SIGSEGV", () => {}), Error, - msg, + msg + "SIGSEGV", ); }, ); diff --git a/tests/unit/stat_test.ts b/tests/unit/stat_test.ts index 59831a069fdf39..0609035b41779a 100644 --- a/tests/unit/stat_test.ts +++ b/tests/unit/stat_test.ts @@ -31,6 +31,13 @@ Deno.test( assert( tempInfo.birthtime === null || now - tempInfo.birthtime.valueOf() < 1000, ); + assert(tempInfo.ctime !== null && now - tempInfo.ctime.valueOf() < 1000); + const mode = tempInfo.mode! & 0o777; + if (Deno.build.os === "windows") { + assertEquals(mode, 0o666); + } else { + assertEquals(mode, 0o600); + } const readmeInfoByUrl = Deno.statSync(pathToAbsoluteFileUrl("README.md")); assert(readmeInfoByUrl.isFile); @@ -65,6 +72,10 @@ Deno.test( tempInfoByUrl.birthtime === null || now - tempInfoByUrl.birthtime.valueOf() < 1000, ); + assert( + tempInfoByUrl.ctime !== null && + now - tempInfoByUrl.ctime.valueOf() < 1000, + ); Deno.removeSync(tempFile, { recursive: true }); Deno.removeSync(tempFileForUrl, { recursive: true }); @@ -171,6 +182,7 @@ Deno.test( assert( tempInfo.birthtime === null || now - tempInfo.birthtime.valueOf() < 1000, ); + assert(tempInfo.ctime !== null && now - tempInfo.ctime.valueOf() < 1000); const tempFileForUrl = await Deno.makeTempFile(); const tempInfoByUrl = await Deno.stat( @@ -191,7 +203,10 @@ Deno.test( tempInfoByUrl.birthtime === null || now - tempInfoByUrl.birthtime.valueOf() < 1000, ); - + assert( + tempInfoByUrl.ctime !== null && + now - tempInfoByUrl.ctime.valueOf() < 1000, + ); Deno.removeSync(tempFile, { recursive: true }); Deno.removeSync(tempFileForUrl, { recursive: true }); }, @@ -271,7 +286,6 @@ Deno.test( const s = Deno.statSync(filename); assert(s.dev !== 0); assert(s.ino === null); - assert(s.mode === null); assert(s.nlink === null); assert(s.uid === null); assert(s.gid === null); diff --git a/tests/unit/streams_test.ts b/tests/unit/streams_test.ts index b866fa7d51488e..53225a1553c6a3 100644 --- a/tests/unit/streams_test.ts +++ b/tests/unit/streams_test.ts @@ -1,5 +1,10 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import { assertEquals, assertRejects, fail } from "./test_util.ts"; +import { + assertEquals, + assertRejects, + assertThrows, + fail, +} from "./test_util.ts"; const { core, @@ -533,3 +538,12 @@ Deno.test(async function decompressionStreamInvalidGzipStillReported() { "corrupt gzip stream does not have a matching checksum", ); }); + +Deno.test(function readableStreamFromWithStringThrows() { + assertThrows( + // @ts-expect-error: primitives are not acceptable + () => ReadableStream.from("string"), + TypeError, + "Failed to execute 'ReadableStream.from': Argument 1 can not be converted to async iterable.", + ); +}); diff --git a/tests/unit/tls_test.ts b/tests/unit/tls_test.ts index aba4d254ce0602..219f4a450831a6 100644 --- a/tests/unit/tls_test.ts +++ b/tests/unit/tls_test.ts @@ -81,7 +81,7 @@ Deno.test( // `Deno.startTls` cannot consume the connection. await assertRejects( () => Deno.startTls(clientConn, { hostname }), - Deno.errors.BadResource, + Deno.errors.Busy, ); serverConn.close(); diff --git a/tests/unit/urlpattern_test.ts b/tests/unit/urlpattern_test.ts index 65c2241737441f..3c1fb0cf19a55f 100644 --- a/tests/unit/urlpattern_test.ts +++ b/tests/unit/urlpattern_test.ts @@ -63,9 +63,3 @@ Deno.test(function urlPatternWithPrototypePollution() { RegExp.prototype.exec = originalExec; } }); - -Deno.test(function urlPatternEmptyFallback() { - const p = new URLPattern({ pathname: "/foo/bar{/:qaz}?" }); - const match = p.exec("https://example.com/foo/bar"); - assertEquals(match!.pathname.groups.qaz, ""); -}); diff --git a/tests/unit/wasm_test.ts b/tests/unit/wasm_test.ts index e0db41ed0edb62..8ee9392f934c36 100644 --- a/tests/unit/wasm_test.ts +++ b/tests/unit/wasm_test.ts @@ -32,7 +32,7 @@ Deno.test(async function wasmInstantiateWorksWithBuffer() { }); // V8's default implementation of `WebAssembly.instantiateStreaming()` if you -// don't set the WASM streaming callback, is to take a byte source. Here we +// don't set the Wasm streaming callback, is to take a byte source. Here we // check that our implementation of the callback disallows it. Deno.test( async function wasmInstantiateStreamingFailsWithBuffer() { @@ -95,7 +95,7 @@ Deno.test( Deno.test( { permissions: { net: true } }, async function wasmStreamingNonTrivial() { - // deno-dom's WASM file is a real-world non-trivial case that gave us + // deno-dom's Wasm file is a real-world non-trivial case that gave us // trouble when implementing this. await WebAssembly.instantiateStreaming(fetch( "http://localhost:4545/assets/deno_dom_0.1.3-alpha2.wasm", diff --git a/tests/unit/webcrypto_test.ts b/tests/unit/webcrypto_test.ts index 57aa19eaee47b5..09552a05871b91 100644 --- a/tests/unit/webcrypto_test.ts +++ b/tests/unit/webcrypto_test.ts @@ -2045,3 +2045,43 @@ Deno.test(async function p521Generate() { assert(key.privateKey instanceof CryptoKey); assert(key.publicKey instanceof CryptoKey); }); + +Deno.test(async function x25519SharedSecret() { + const alicesKeyPair = await crypto.subtle.generateKey( + { + name: "X25519", + }, + false, + ["deriveBits"], + ) as CryptoKeyPair; + + const bobsKeyPair = await crypto.subtle.generateKey( + { + name: "X25519", + }, + false, + ["deriveBits"], + ) as CryptoKeyPair; + + const sharedSecret1 = await crypto.subtle.deriveBits( + { + name: "X25519", + public: bobsKeyPair.publicKey, + }, + alicesKeyPair.privateKey, + 128, + ); + + const sharedSecret2 = await crypto.subtle.deriveBits( + { + name: "X25519", + public: alicesKeyPair.publicKey, + }, + bobsKeyPair.privateKey, + 128, + ); + + assertEquals(sharedSecret1.byteLength, sharedSecret2.byteLength); + assertEquals(sharedSecret1.byteLength, 16); + assertEquals(new Uint8Array(sharedSecret1), new Uint8Array(sharedSecret2)); +}); diff --git a/tests/unit/webgpu_test.ts b/tests/unit/webgpu_test.ts index 6c91abe4ade1b5..aac75d34206d76 100644 --- a/tests/unit/webgpu_test.ts +++ b/tests/unit/webgpu_test.ts @@ -553,6 +553,57 @@ Deno.test({ device.destroy(); }); +Deno.test({ + ignore: isWsl || isCIWithoutGPU, +}, async function adapterLimitsAreNumbers() { + const limitNames = [ + "maxTextureDimension1D", + "maxTextureDimension2D", + "maxTextureDimension3D", + "maxTextureArrayLayers", + "maxBindGroups", + "maxDynamicUniformBuffersPerPipelineLayout", + "maxDynamicStorageBuffersPerPipelineLayout", + "maxSampledTexturesPerShaderStage", + "maxSamplersPerShaderStage", + "maxStorageBuffersPerShaderStage", + "maxStorageTexturesPerShaderStage", + "maxUniformBuffersPerShaderStage", + "maxUniformBufferBindingSize", + "maxStorageBufferBindingSize", + "minUniformBufferOffsetAlignment", + "minStorageBufferOffsetAlignment", + "maxVertexBuffers", + "maxVertexAttributes", + "maxVertexBufferArrayStride", + "maxInterStageShaderComponents", + "maxComputeWorkgroupStorageSize", + "maxComputeInvocationsPerWorkgroup", + "maxComputeWorkgroupSizeX", + "maxComputeWorkgroupSizeY", + "maxComputeWorkgroupSizeZ", + "maxComputeWorkgroupsPerDimension", + ]; + + const adapter = await navigator.gpu.requestAdapter(); + assert(adapter); + + for (const limitName of limitNames) { + // deno-lint-ignore ban-ts-comment + // @ts-ignore + assertEquals(typeof adapter.limits[limitName], "number"); + } + + const device = await adapter.requestDevice({ + // deno-lint-ignore ban-ts-comment + // @ts-ignore + requiredLimits: adapter.limits, + }); + assert(device); + + device.destroy(); +}); + async function checkIsWsl() { return Deno.build.os === "linux" && await hasMicrosoftProcVersion(); diff --git a/tests/unit/websocket_test.ts b/tests/unit/websocket_test.ts index 7db876b1774a45..d9878828db4d97 100644 --- a/tests/unit/websocket_test.ts +++ b/tests/unit/websocket_test.ts @@ -453,7 +453,8 @@ Deno.test("invalid server", async () => { const { promise, resolve } = Promise.withResolvers(); const ws = new WebSocket("ws://localhost:2121"); let err = false; - ws.onerror = () => { + ws.onerror = (e) => { + assert("error" in e); err = true; }; ws.onclose = () => { @@ -805,3 +806,18 @@ Deno.test("Close connection", async () => { await server.finished; conn.close(); }); + +Deno.test("send to a closed socket", async () => { + const { promise, resolve } = Promise.withResolvers(); + const ws = new WebSocket("ws://localhost:4242"); + const blob = new Blob(["foo"]); + ws.onerror = () => fail(); + ws.onopen = () => { + ws.close(); + ws.send(blob); + }; + ws.onclose = () => { + resolve(); + }; + await promise; +}); diff --git a/tests/unit/webstorage_test.ts b/tests/unit/webstorage_test.ts index 9dc560af1b507b..aa832b1c4b2e37 100644 --- a/tests/unit/webstorage_test.ts +++ b/tests/unit/webstorage_test.ts @@ -50,3 +50,8 @@ Deno.test(function webstorageProxy() { assertEquals(localStorage[symbol as any], "bar"); assertEquals(symbol in localStorage, true); }); + +Deno.test(function webstorageGetOwnPropertyDescriptorSymbol() { + localStorage.clear(); + Object.getOwnPropertyDescriptor(localStorage, Symbol("foo")); +}); diff --git a/tests/unit_node/_fs/_fs_handle_test.ts b/tests/unit_node/_fs/_fs_handle_test.ts index 755e091fd7a505..e26b82aa067f16 100644 --- a/tests/unit_node/_fs/_fs_handle_test.ts +++ b/tests/unit_node/_fs/_fs_handle_test.ts @@ -93,3 +93,27 @@ Deno.test("[node/fs filehandle.write] Write from string", async function () { assertEquals(res.bytesWritten, 11); assertEquals(decoder.decode(data), "hello world"); }); + +Deno.test("[node/fs filehandle.stat] Get file status", async function () { + const fileHandle = await fs.open(testData); + const stat = await fileHandle.stat(); + + assertEquals(stat.isFile(), true); + assertEquals(stat.size, "hello world".length); + + await fileHandle.close(); +}); + +Deno.test("[node/fs filehandle.writeFile] Write to file", async function () { + const tempFile: string = await Deno.makeTempFile(); + const fileHandle = await fs.open(tempFile, "w"); + + const str = "hello world"; + await fileHandle.writeFile(str); + + const data = Deno.readFileSync(tempFile); + await Deno.remove(tempFile); + await fileHandle.close(); + + assertEquals(decoder.decode(data), "hello world"); +}); diff --git a/tests/unit_node/_fs/_fs_readFile_test.ts b/tests/unit_node/_fs/_fs_readFile_test.ts index ea36b9d866af4d..a75f12d1f6fe39 100644 --- a/tests/unit_node/_fs/_fs_readFile_test.ts +++ b/tests/unit_node/_fs/_fs_readFile_test.ts @@ -2,7 +2,7 @@ import { assertCallbackErrorUncaught } from "../_test_utils.ts"; import { promises, readFile, readFileSync } from "node:fs"; import * as path from "@std/path"; -import { assert, assertEquals } from "@std/assert"; +import { assert, assertEquals, assertMatch } from "@std/assert"; const moduleDir = path.dirname(path.fromFileUrl(import.meta.url)); const testData = path.resolve(moduleDir, "testdata", "hello.txt"); @@ -121,3 +121,26 @@ Deno.test("fs.promises.readFile with no arg call rejects with error correctly", // @ts-ignore no arg call needs to be supported await promises.readFile().catch((_e) => {}); }); + +Deno.test("fs.readFile error message contains path + syscall", async () => { + const path = "/does/not/exist"; + const err = await new Promise((resolve) => { + readFile(path, "utf-8", (err) => resolve(err)); + }); + if (err instanceof Error) { + assert(err.message.includes(path), "Path not found in error message"); + assertMatch(err.message, /[,\s]open\s/); + } +}); + +Deno.test("fs.readFileSync error message contains path + syscall", () => { + const path = "/does/not/exist"; + try { + readFileSync(path, "utf-8"); + } catch (err) { + if (err instanceof Error) { + assert(err.message.includes(path), "Path not found in error message"); + assertMatch(err.message, /[,\s]open\s/); + } + } +}); diff --git a/tests/unit_node/_fs/_fs_stat_test.ts b/tests/unit_node/_fs/_fs_stat_test.ts index 02c620e2dc141f..3cbbe940b02041 100644 --- a/tests/unit_node/_fs/_fs_stat_test.ts +++ b/tests/unit_node/_fs/_fs_stat_test.ts @@ -1,7 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. import { assertCallbackErrorUncaught } from "../_test_utils.ts"; import { BigIntStats, stat, Stats, statSync } from "node:fs"; -import { assertEquals, fail } from "@std/assert"; +import { assert, assertEquals, fail } from "@std/assert"; export function assertStats(actual: Stats, expected: Deno.FileInfo) { assertEquals(actual.dev, expected.dev); @@ -18,9 +18,11 @@ export function assertStats(actual: Stats, expected: Deno.FileInfo) { assertEquals(actual.atime?.getTime(), expected.atime?.getTime()); assertEquals(actual.mtime?.getTime(), expected.mtime?.getTime()); assertEquals(actual.birthtime?.getTime(), expected.birthtime?.getTime()); + assertEquals(actual.ctime?.getTime(), expected.ctime?.getTime()); assertEquals(actual.atimeMs ?? undefined, expected.atime?.getTime()); assertEquals(actual.mtimeMs ?? undefined, expected.mtime?.getTime()); assertEquals(actual.birthtimeMs ?? undefined, expected.birthtime?.getTime()); + assertEquals(actual.ctimeMs ?? undefined, expected.ctime?.getTime()); assertEquals(actual.isFile(), expected.isFile); assertEquals(actual.isDirectory(), expected.isDirectory); assertEquals(actual.isSymbolicLink(), expected.isSymlink); @@ -49,6 +51,7 @@ export function assertStatsBigInt( assertEquals(actual.atime?.getTime(), expected.atime?.getTime()); assertEquals(actual.mtime?.getTime(), expected.mtime?.getTime()); assertEquals(actual.birthtime?.getTime(), expected.birthtime?.getTime()); + assertEquals(actual.ctime?.getTime(), expected.ctime?.getTime()); assertEquals( actual.atimeMs === null ? undefined : Number(actual.atimeMs), expected.atime?.getTime(), @@ -61,6 +64,10 @@ export function assertStatsBigInt( actual.birthtimeMs === null ? undefined : Number(actual.birthtimeMs), expected.birthtime?.getTime(), ); + assertEquals( + actual.ctimeMs === null ? undefined : Number(actual.ctimeMs), + expected.ctime?.getTime(), + ); assertEquals(actual.atimeNs === null, actual.atime === null); assertEquals(actual.mtimeNs === null, actual.mtime === null); assertEquals(actual.birthtimeNs === null, actual.birthtime === null); @@ -145,3 +152,38 @@ Deno.test({ assertEquals(stats.isSocket(), false); }, }); + +Deno.test({ + name: "[node/fs] stat invalid path error", + async fn() { + try { + await new Promise((resolve, reject) => { + stat( + // deno-lint-ignore no-explicit-any + undefined as any, + (err, stats) => err ? reject(err) : resolve(stats), + ); + }); + fail(); + } catch (err) { + assert(err instanceof TypeError); + // deno-lint-ignore no-explicit-any + assertEquals((err as any).code, "ERR_INVALID_ARG_TYPE"); + } + }, +}); + +Deno.test({ + name: "[node/fs] statSync invalid path error", + fn() { + try { + // deno-lint-ignore no-explicit-any + statSync(undefined as any); + fail(); + } catch (err) { + assert(err instanceof TypeError); + // deno-lint-ignore no-explicit-any + assertEquals((err as any).code, "ERR_INVALID_ARG_TYPE"); + } + }, +}); diff --git a/tests/unit_node/child_process_test.ts b/tests/unit_node/child_process_test.ts index f776fa4acd5925..0ea3c46cf0c03c 100644 --- a/tests/unit_node/child_process_test.ts +++ b/tests/unit_node/child_process_test.ts @@ -1045,3 +1045,19 @@ Deno.test(async function sendAfterClosedThrows() { await timeout.promise; }); + +Deno.test(async function noWarningsFlag() { + const code = ``; + const file = await Deno.makeTempFile(); + await Deno.writeTextFile(file, code); + const timeout = withTimeout(); + const child = CP.fork(file, [], { + execArgv: ["--no-warnings"], + stdio: ["inherit", "inherit", "inherit", "ipc"], + }); + child.on("close", () => { + timeout.resolve(); + }); + + await timeout.promise; +}); diff --git a/tests/unit_node/crypto/crypto_key_test.ts b/tests/unit_node/crypto/crypto_key_test.ts index 7995ce5d3f8d17..5d206acc723fde 100644 --- a/tests/unit_node/crypto/crypto_key_test.ts +++ b/tests/unit_node/crypto/crypto_key_test.ts @@ -656,3 +656,47 @@ z6TExWlQMjt66nV7R8cRAkzmABrG+NW3e8Zpac7Lkuv+zu0S+K7c assertEquals(publicKey.type, "public"); assertEquals(publicKey.asymmetricKeyType, "rsa"); }); + +// https://github.com/denoland/deno/issues/26188 +Deno.test("generateKeyPair large pem", function () { + const passphrase = "mypassphrase"; + const cipher = "aes-256-cbc"; + const modulusLength = 4096; + + generateKeyPairSync("rsa", { + modulusLength, + publicKeyEncoding: { + type: "spki", + format: "pem", + }, + privateKeyEncoding: { + type: "pkcs8", + format: "pem", + cipher, + passphrase, + }, + }); +}); + +Deno.test("generateKeyPair promisify", async () => { + const passphrase = "mypassphrase"; + const cipher = "aes-256-cbc"; + const modulusLength = 4096; + + const { privateKey, publicKey } = await promisify(generateKeyPair)("rsa", { + modulusLength, + publicKeyEncoding: { + type: "spki", + format: "pem", + }, + privateKeyEncoding: { + type: "pkcs8", + format: "pem", + cipher, + passphrase, + }, + }); + + assert(publicKey.startsWith("-----BEGIN PUBLIC KEY-----")); + assert(privateKey.startsWith("-----BEGIN PRIVATE KEY-----")); +}); diff --git a/tests/unit_node/crypto/crypto_misc_test.ts b/tests/unit_node/crypto/crypto_misc_test.ts index 47a48b1bf8d179..007009339d423b 100644 --- a/tests/unit_node/crypto/crypto_misc_test.ts +++ b/tests/unit_node/crypto/crypto_misc_test.ts @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import { randomFillSync, randomUUID } from "node:crypto"; +import { randomFillSync, randomUUID, timingSafeEqual } from "node:crypto"; +import { Buffer } from "node:buffer"; import { assert, assertEquals } from "../../unit/test_util.ts"; import { assertNotEquals } from "@std/assert"; @@ -28,3 +29,10 @@ Deno.test("[node/crypto.randomFillSync] array buffer view", () => { assertEquals(buf.subarray(0, 8), new Uint8Array(8)); assertEquals(buf.subarray(24, 32), new Uint8Array(8)); }); + +Deno.test("[node/crypto.timingSafeEqual] compares equal Buffer with different byteOffset", () => { + const a = Buffer.from([212, 213]); + const b = Buffer.from([0, 0, 0, 0, 0, 0, 0, 0, 212, 213]).subarray(8); + + assert(timingSafeEqual(a, b)); +}); diff --git a/tests/unit_node/fetch_test.ts b/tests/unit_node/fetch_test.ts new file mode 100644 index 00000000000000..399d6052a56377 --- /dev/null +++ b/tests/unit_node/fetch_test.ts @@ -0,0 +1,18 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +import { assertEquals } from "@std/assert"; +import { createReadStream } from "node:fs"; + +Deno.test("fetch node stream", async () => { + const file = createReadStream("tests/testdata/assets/fixture.json"); + + const response = await fetch("http://localhost:4545/echo_server", { + method: "POST", + body: file, + }); + + assertEquals( + await response.text(), + await Deno.readTextFile("tests/testdata/assets/fixture.json"), + ); +}); diff --git a/tests/unit_node/fs_test.ts b/tests/unit_node/fs_test.ts index b1f7c53e8d5160..32bea40e75ae74 100644 --- a/tests/unit_node/fs_test.ts +++ b/tests/unit_node/fs_test.ts @@ -1,25 +1,33 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import { assert, assertEquals, assertThrows } from "@std/assert"; +/// +import { assert, assertEquals, assertRejects, assertThrows } from "@std/assert"; import { join } from "node:path"; +import { fileURLToPath } from "node:url"; import { tmpdir } from "node:os"; import { + closeSync, constants, + copyFileSync, createWriteStream, existsSync, lstatSync, mkdtempSync, + openSync, promises, readFileSync, + readSync, Stats, statSync, writeFileSync, } from "node:fs"; import { constants as fsPromiseConstants, + copyFile, cp, FileHandle, open, + stat, writeFile, } from "node:fs/promises"; import process from "node:process"; @@ -117,6 +125,50 @@ Deno.test( }, ); +Deno.test( + "[node/fs statSync] throw error with path information", + () => { + const file = "non-exist-file"; + const fileUrl = new URL(file, import.meta.url); + + assertThrows(() => { + statSync(file); + }, "Error: ENOENT: no such file or directory, stat 'non-exist-file'"); + + assertThrows(() => { + statSync(fileUrl); + }, `Error: ENOENT: no such file or directory, stat '${fileUrl.pathname}'`); + }, +); + +Deno.test( + "[node/fs/promises stat] throw error with path information", + async () => { + const file = "non-exist-file"; + const fileUrl = new URL(file, import.meta.url); + + try { + await stat(file); + } catch (error: unknown) { + assertEquals( + `${error}`, + "Error: ENOENT: no such file or directory, stat 'non-exist-file'", + ); + } + + try { + await stat(fileUrl); + } catch (error: unknown) { + assertEquals( + `${error}`, + `Error: ENOENT: no such file or directory, stat '${ + fileURLToPath(fileUrl) + }'`, + ); + } + }, +); + Deno.test( "[node/fs/promises cp] copy file", async () => { @@ -201,3 +253,40 @@ Deno.test( assertEquals(res, [0, 1, 2, 3, 4, 5]); }, ); + +Deno.test("[node/fs] readSync works", () => { + const fd = openSync("tests/testdata/assets/hello.txt", "r"); + const buf = new Uint8Array(256); + const bytesRead = readSync(fd!, buf); + assertEquals(bytesRead, 12); + closeSync(fd!); +}); + +Deno.test("[node/fs] copyFile COPYFILE_EXCL works", async () => { + const dir = mkdtempSync(join(tmpdir(), "foo-")); + const src = join(dir, "src.txt"); + const dest = join(dir, "dest.txt"); + await writeFile(src, ""); + await copyFile(src, dest, fsPromiseConstants.COPYFILE_EXCL); + assert(existsSync(dest)); + await assertRejects(() => + copyFile(src, dest, fsPromiseConstants.COPYFILE_EXCL) + ); + const dest2 = join(dir, "dest2.txt"); + copyFileSync(src, dest2, fsPromiseConstants.COPYFILE_EXCL); + assert(existsSync(dest2)); + assertThrows(() => + copyFileSync(src, dest2, fsPromiseConstants.COPYFILE_EXCL) + ); +}); + +Deno.test("[node/fs] statSync throws ENOENT for invalid path containing colon in it", () => { + // deno-lint-ignore no-explicit-any + const err: any = assertThrows(() => { + // Note: Deno.stat throws ERROR_INVALID_NAME (os error 123) instead of + // ERROR_FILE_NOT_FOUND (os error 2) on windows. This case checks that + // ERROR_INVALID_NAME is mapped to ENOENT correctly on node compat layer. + statSync("jsr:@std/assert"); + }); + assertEquals(err.code, "ENOENT"); +}); diff --git a/tests/unit_node/http2_test.ts b/tests/unit_node/http2_test.ts index cb939646be2524..c540c90f7e56e0 100644 --- a/tests/unit_node/http2_test.ts +++ b/tests/unit_node/http2_test.ts @@ -10,7 +10,12 @@ import * as net from "node:net"; import { assert, assertEquals } from "@std/assert"; import { curlRequest } from "../unit/test_util.ts"; -for (const url of ["http://127.0.0.1:4246", "https://127.0.0.1:4247"]) { +// Increase the timeout for the auto select family to avoid flakiness +net.setDefaultAutoSelectFamilyAttemptTimeout( + net.getDefaultAutoSelectFamilyAttemptTimeout() * 30, +); + +for (const url of ["http://localhost:4246", "https://localhost:4247"]) { Deno.test(`[node/http2 client] ${url}`, { ignore: Deno.build.os === "windows", }, async () => { @@ -155,7 +160,7 @@ Deno.test("[node/http2.createServer()]", { res.end(); }); server.listen(0); - const port = ( server.address()).port; + const port = (server.address() as net.AddressInfo).port; const endpoint = `http://localhost:${port}`; const response = await curlRequest([ diff --git a/tests/unit_node/http_test.ts b/tests/unit_node/http_test.ts index f85b1466b5bc3b..31ac6bee25e88b 100644 --- a/tests/unit_node/http_test.ts +++ b/tests/unit_node/http_test.ts @@ -3,10 +3,14 @@ // deno-lint-ignore-file no-console import EventEmitter from "node:events"; -import http, { type RequestOptions, type ServerResponse } from "node:http"; +import http, { + IncomingMessage, + type RequestOptions, + ServerResponse, +} from "node:http"; import url from "node:url"; import https from "node:https"; -import net from "node:net"; +import net, { Socket } from "node:net"; import fs from "node:fs"; import { text } from "node:stream/consumers"; @@ -1143,6 +1147,35 @@ Deno.test("[node/http] ServerResponse appendHeader set-cookie", async () => { await promise; }); +Deno.test("[node/http] ServerResponse header names case insensitive", async () => { + const { promise, resolve } = Promise.withResolvers(); + const server = http.createServer((_req, res) => { + res.setHeader("Content-Length", "12345"); + assert(res.hasHeader("Content-Length")); + res.removeHeader("content-length"); + assertEquals(res.getHeader("Content-Length"), undefined); + assert(!res.hasHeader("Content-Length")); + res.appendHeader("content-length", "12345"); + res.removeHeader("Content-Length"); + assertEquals(res.getHeader("content-length"), undefined); + assert(!res.hasHeader("content-length")); + res.end("Hello World"); + }); + + server.listen(async () => { + const { port } = server.address() as { port: number }; + const res = await fetch(`http://localhost:${port}`); + assertEquals(res.headers.get("Content-Length"), null); + assertEquals(res.headers.get("content-length"), null); + assertEquals(await res.text(), "Hello World"); + server.close(() => { + resolve(); + }); + }); + + await promise; +}); + Deno.test("[node/http] IncomingMessage override", () => { const req = new http.IncomingMessage(new net.Socket()); // https://github.com/dougmoscrop/serverless-http/blob/3aaa6d0fe241109a8752efb011c242d249f32368/lib/request.js#L20-L30 @@ -1672,3 +1705,75 @@ Deno.test("[node/http] upgraded socket closes when the server closed without clo await clientSocketClosed.promise; await serverProcessClosed.promise; }); + +// deno-lint-ignore require-await +Deno.test("[node/http] ServerResponse.call()", async () => { + function Wrapper(this: unknown, req: IncomingMessage) { + ServerResponse.call(this, req); + } + Object.setPrototypeOf(Wrapper.prototype, ServerResponse.prototype); + + // deno-lint-ignore no-explicit-any + const wrapper = new (Wrapper as any)(new IncomingMessage(new Socket())); + + assert(wrapper instanceof ServerResponse); +}); + +Deno.test("[node/http] ServerResponse _header", async () => { + const { promise, resolve } = Promise.withResolvers(); + const server = http.createServer((_req, res) => { + assert(Object.hasOwn(res, "_header")); + res.end(); + }); + + server.listen(async () => { + const { port } = server.address() as { port: number }; + const res = await fetch(`http://localhost:${port}`); + await res.body?.cancel(); + server.close(() => { + resolve(); + }); + }); + + await promise; +}); + +Deno.test("[node/http] ServerResponse connection", async () => { + const { promise, resolve } = Promise.withResolvers(); + const server = http.createServer((_req, res) => { + assert(Object.hasOwn(res, "connection")); + assert(res.connection instanceof Socket); + res.end(); + }); + + server.listen(async () => { + const { port } = server.address() as { port: number }; + const res = await fetch(`http://localhost:${port}`); + await res.body?.cancel(); + server.close(() => { + resolve(); + }); + }); + + await promise; +}); + +Deno.test("[node/http] ServerResponse socket", async () => { + const { promise, resolve } = Promise.withResolvers(); + const server = http.createServer((_req, res) => { + assert(Object.hasOwn(res, "socket")); + assert(res.socket instanceof Socket); + res.end(); + }); + + server.listen(async () => { + const { port } = server.address() as { port: number }; + const res = await fetch(`http://localhost:${port}`); + await res.body?.cancel(); + server.close(() => { + resolve(); + }); + }); + + await promise; +}); diff --git a/tests/unit_node/perf_hooks_test.ts b/tests/unit_node/perf_hooks_test.ts index d5b9000410b07e..83d0062228ee4d 100644 --- a/tests/unit_node/perf_hooks_test.ts +++ b/tests/unit_node/perf_hooks_test.ts @@ -1,7 +1,11 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. import * as perfHooks from "node:perf_hooks"; -import { performance, PerformanceObserver } from "node:perf_hooks"; -import { assertEquals, assertThrows } from "@std/assert"; +import { + monitorEventLoopDelay, + performance, + PerformanceObserver, +} from "node:perf_hooks"; +import { assert, assertEquals, assertThrows } from "@std/assert"; Deno.test({ name: "[perf_hooks] performance", @@ -68,3 +72,17 @@ Deno.test("[perf_hooks]: eventLoopUtilization", () => { assertEquals(typeof obj.active, "number"); assertEquals(typeof obj.utilization, "number"); }); + +Deno.test("[perf_hooks]: monitorEventLoopDelay", async () => { + const e = monitorEventLoopDelay(); + assertEquals(e.count, 0); + e.enable(); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + assert(e.min > 0); + assert(e.minBigInt > 0n); + assert(e.count > 0); + + e.disable(); +}); diff --git a/tests/unit_node/process_test.ts b/tests/unit_node/process_test.ts index add9e1280a46b2..49de2dce1dd7e2 100644 --- a/tests/unit_node/process_test.ts +++ b/tests/unit_node/process_test.ts @@ -25,7 +25,6 @@ import { assertThrows, fail, } from "@std/assert"; -import { assertSpyCall, assertSpyCalls, spy } from "@std/testing/mock"; import { stripAnsiCode } from "@std/fmt/colors"; import * as path from "@std/path"; import { delay } from "@std/async/delay"; @@ -239,33 +238,6 @@ Deno.test({ }, }); -Deno.test({ - name: "process.on - ignored signals on windows", - ignore: Deno.build.os !== "windows", - fn() { - const ignoredSignals = ["SIGHUP", "SIGUSR1", "SIGUSR2"]; - - for (const signal of ignoredSignals) { - using consoleSpy = spy(console, "warn"); - const handler = () => {}; - process.on(signal, handler); - process.off(signal, handler); - assertSpyCall(consoleSpy, 0, { - args: [`Ignoring signal "${signal}" on Windows`], - }); - } - - { - using consoleSpy = spy(console, "warn"); - const handler = () => {}; - process.on("SIGTERM", handler); - process.off("SIGTERM", handler); - // No warning is made for SIGTERM - assertSpyCalls(consoleSpy, 0); - } - }, -}); - Deno.test( { permissions: { run: true, read: true } }, async function processKill() { @@ -691,6 +663,16 @@ Deno.test({ assertStrictEquals(process.stdout.clearLine, undefined); assertStrictEquals(process.stdout.clearScreenDown, undefined); } + + // Allows overwriting `process.stdout.isTTY` + // https://github.com/denoland/deno/issues/26123 + const original = process.stdout.isTTY; + try { + process.stdout.isTTY = !isTTY; + assertEquals(process.stdout.isTTY, !isTTY); + } finally { + process.stdout.isTTY = original; + } }, }); @@ -1165,3 +1147,14 @@ Deno.test("process.cpuUsage()", () => { assert(typeof cpuUsage.user === "number"); assert(typeof cpuUsage.system === "number"); }); + +Deno.test("process.stdout.columns writable", () => { + process.stdout.columns = 80; + assertEquals(process.stdout.columns, 80); +}); + +Deno.test("getBuiltinModule", () => { + assert(process.getBuiltinModule("fs")); + assert(process.getBuiltinModule("node:fs")); + assertEquals(process.getBuiltinModule("something"), undefined); +}); diff --git a/tests/unit_node/timers_test.ts b/tests/unit_node/timers_test.ts index 868ba21d64b2c3..ecff32e763b247 100644 --- a/tests/unit_node/timers_test.ts +++ b/tests/unit_node/timers_test.ts @@ -3,6 +3,7 @@ import { assert, fail } from "@std/assert"; import * as timers from "node:timers"; import * as timersPromises from "node:timers/promises"; +import { assertEquals } from "@std/assert"; Deno.test("[node/timers setTimeout]", () => { { @@ -99,6 +100,16 @@ Deno.test("[node/timers refresh cancelled timer]", () => { p.refresh(); }); +Deno.test("[node/timers] clearTimeout with number", () => { + const timer = +timers.setTimeout(() => fail(), 10); + timers.clearTimeout(timer); +}); + +Deno.test("[node/timers] clearInterval with number", () => { + const timer = +timers.setInterval(() => fail(), 10); + timers.clearInterval(timer); +}); + Deno.test("[node/timers setImmediate returns Immediate object]", () => { const { clearImmediate, setImmediate } = timers; @@ -108,3 +119,154 @@ Deno.test("[node/timers setImmediate returns Immediate object]", () => { imm.hasRef(); clearImmediate(imm); }); + +Deno.test({ + name: "setInterval yields correct values at expected intervals", + async fn() { + // Test configuration + const CONFIG = { + expectedValue: 42, + intervalMs: 100, + iterations: 3, + tolerancePercent: Deno.env.get("CI") != null ? 75 : 50, + }; + + const { setInterval } = timersPromises; + const results: Array<{ value: number; timestamp: number }> = []; + const startTime = Date.now(); + + const iterator = setInterval(CONFIG.intervalMs, CONFIG.expectedValue); + + for await (const value of iterator) { + results.push({ + value, + timestamp: Date.now(), + }); + if (results.length === CONFIG.iterations) { + break; + } + } + + const values = results.map((r) => r.value); + assertEquals( + values, + Array(CONFIG.iterations).fill(CONFIG.expectedValue), + `Each iteration should yield ${CONFIG.expectedValue}`, + ); + + const intervals = results.slice(1).map((result, index) => ({ + interval: result.timestamp - results[index].timestamp, + iterationNumber: index + 1, + })); + + const toleranceMs = (CONFIG.tolerancePercent / 100) * CONFIG.intervalMs; + const expectedRange = { + min: CONFIG.intervalMs - toleranceMs, + max: CONFIG.intervalMs + toleranceMs, + }; + + intervals.forEach(({ interval, iterationNumber }) => { + const isWithinTolerance = interval >= expectedRange.min && + interval <= expectedRange.max; + + assertEquals( + isWithinTolerance, + true, + `Iteration ${iterationNumber}: Interval ${interval}ms should be within ` + + `${expectedRange.min}ms and ${expectedRange.max}ms ` + + `(${CONFIG.tolerancePercent}% tolerance of ${CONFIG.intervalMs}ms)`, + ); + }); + + const totalDuration = results[results.length - 1].timestamp - startTime; + const expectedDuration = CONFIG.intervalMs * CONFIG.iterations; + const isDurationReasonable = + totalDuration >= (expectedDuration - toleranceMs) && + totalDuration <= (expectedDuration + toleranceMs); + + assertEquals( + isDurationReasonable, + true, + `Total duration ${totalDuration}ms should be close to ${expectedDuration}ms ` + + `(within ${toleranceMs}ms tolerance)`, + ); + + const timestamps = results.map((r) => r.timestamp); + const areTimestampsOrdered = timestamps.every((timestamp, i) => + i === 0 || timestamp > timestamps[i - 1] + ); + + assertEquals( + areTimestampsOrdered, + true, + "Timestamps should be strictly increasing", + ); + }, +}); + +Deno.test({ + name: "setInterval with AbortSignal stops after expected duration", + async fn() { + const INTERVAL_MS = 500; + const TOTAL_DURATION_MS = 3000; + const TOLERANCE_MS = 500; + const DELTA_TOLERANCE_MS = Deno.env.get("CI") != null ? 100 : 50; + + const abortController = new AbortController(); + const { setInterval } = timersPromises; + + // Set up abort after specified duration + const abortTimeout = timers.setTimeout(() => { + abortController.abort(); + }, TOTAL_DURATION_MS); + + // Track iterations and timing + const startTime = Date.now(); + const iterations: number[] = []; + + try { + for await ( + const _timestamp of setInterval(INTERVAL_MS, undefined, { + signal: abortController.signal, + }) + ) { + iterations.push(Date.now() - startTime); + } + } catch (error) { + if (error instanceof Error && error.name !== "AbortError") { + throw error; + } + } finally { + timers.clearTimeout(abortTimeout); + } + + // Validate timing + const totalDuration = iterations[iterations.length - 1]; + const isWithinTolerance = + totalDuration >= (TOTAL_DURATION_MS - TOLERANCE_MS) && + totalDuration <= (TOTAL_DURATION_MS + TOLERANCE_MS); + + assertEquals( + isWithinTolerance, + true, + `Total duration ${totalDuration}ms should be within ±${TOLERANCE_MS}ms of ${TOTAL_DURATION_MS}ms`, + ); + + // Validate interval consistency + const intervalDeltas = iterations.slice(1).map((time, i) => + time - iterations[i] + ); + + intervalDeltas.forEach((delta, i) => { + const isIntervalValid = delta >= (INTERVAL_MS - DELTA_TOLERANCE_MS) && + delta <= (INTERVAL_MS + DELTA_TOLERANCE_MS); + assertEquals( + isIntervalValid, + true, + `Interval ${ + i + 1 + } duration (${delta}ms) should be within ±${DELTA_TOLERANCE_MS}ms of ${INTERVAL_MS}ms`, + ); + }); + }, +}); diff --git a/tests/unit_node/tls_test.ts b/tests/unit_node/tls_test.ts index 7daa544c74e71c..627b948cd1d8e3 100644 --- a/tests/unit_node/tls_test.ts +++ b/tests/unit_node/tls_test.ts @@ -149,10 +149,12 @@ Deno.test("tls.createServer creates a TLS server", async () => { }, ); server.listen(0, async () => { - const conn = await Deno.connectTls({ - hostname: "127.0.0.1", + const tcpConn = await Deno.connect({ // deno-lint-ignore no-explicit-any port: (server.address() as any).port, + }); + const conn = await Deno.startTls(tcpConn, { + hostname: "localhost", caCerts: [rootCaCert], }); @@ -227,3 +229,45 @@ Deno.test("tls.rootCertificates is not empty", () => { (tls.rootCertificates as string[]).push("new cert"); }, TypeError); }); + +Deno.test("TLSSocket.alpnProtocol is set for client", async () => { + const listener = Deno.listenTls({ + hostname: "localhost", + port: 0, + key, + cert, + alpnProtocols: ["a"], + }); + const outgoing = tls.connect({ + host: "::1", + servername: "localhost", + port: listener.addr.port, + ALPNProtocols: ["a"], + secureContext: { + ca: rootCaCert, + // deno-lint-ignore no-explicit-any + } as any, + }); + + const conn = await listener.accept(); + const handshake = await conn.handshake(); + assertEquals(handshake.alpnProtocol, "a"); + conn.close(); + outgoing.destroy(); + listener.close(); + await new Promise((resolve) => outgoing.on("close", resolve)); +}); + +Deno.test("tls connect upgrade tcp", async () => { + const { promise, resolve } = Promise.withResolvers(); + + const socket = new net.Socket(); + socket.connect(443, "google.com"); + socket.on("connect", () => { + const secure = tls.connect({ socket }); + secure.on("secureConnect", () => resolve()); + }); + + await promise; + socket.destroy(); +}); diff --git a/tests/unit_node/util_test.ts b/tests/unit_node/util_test.ts index edd500262333e3..6267018b126f7c 100644 --- a/tests/unit_node/util_test.ts +++ b/tests/unit_node/util_test.ts @@ -348,3 +348,13 @@ Deno.test("[util] aborted()", async () => { await promise; assertEquals(done, true); }); + +Deno.test("[util] styleText()", () => { + const redText = util.styleText("red", "error"); + assertEquals(redText, "\x1B[31merror\x1B[39m"); +}); + +Deno.test("[util] styleText() with array of formats", () => { + const colored = util.styleText(["red", "green"], "error"); + assertEquals(colored, "\x1b[32m\x1b[31merror\x1b[39m\x1b[39m"); +}); diff --git a/tests/unit_node/worker_threads_test.ts b/tests/unit_node/worker_threads_test.ts index ac797601f5981e..24a910789894c1 100644 --- a/tests/unit_node/worker_threads_test.ts +++ b/tests/unit_node/worker_threads_test.ts @@ -621,3 +621,204 @@ Deno.test({ worker.terminate(); }, }); + +Deno.test({ + name: "[node/worker_threads] receiveMessageOnPort doesn't exit receive loop", + async fn() { + const worker = new workerThreads.Worker( + ` + import { parentPort, receiveMessageOnPort } from "node:worker_threads"; + parentPort.on("message", (msg) => { + const port = msg.port; + port.on("message", (msg2) => { + if (msg2 === "c") { + port.postMessage("done"); + port.unref(); + parentPort.unref(); + } + }); + parentPort.postMessage("ready"); + const msg2 = receiveMessageOnPort(port); + }); + `, + { eval: true }, + ); + + const { port1, port2 } = new workerThreads.MessageChannel(); + + worker.postMessage({ port: port2 }, [port2]); + + const done = Promise.withResolvers(); + + port1.on("message", (msg) => { + assertEquals(msg, "done"); + worker.unref(); + port1.close(); + done.resolve(true); + }); + worker.on("message", (msg) => { + assertEquals(msg, "ready"); + port1.postMessage("a"); + port1.postMessage("b"); + port1.postMessage("c"); + }); + + const timeout = setTimeout(() => { + fail("Test timed out"); + }, 20_000); + try { + const result = await done.promise; + assertEquals(result, true); + } finally { + clearTimeout(timeout); + } + }, +}); + +Deno.test({ + name: "[node/worker_threads] MessagePort.unref doesn't exit receive loop", + async fn() { + const worker = new workerThreads.Worker( + ` + import { parentPort } from "node:worker_threads"; + const assertEquals = (a, b) => { + if (a !== b) { + throw new Error(); + } + }; + let state = 0; + parentPort.on("message", (msg) => { + const port = msg.port; + const expect = ["a", "b", "c"]; + port.on("message", (msg2) => { + assertEquals(msg2, expect[state++]); + if (msg2 === "c") { + port.postMessage({ type: "done", got: msg2 }); + parentPort.unref(); + } + }); + port.unref(); + parentPort.postMessage("ready"); + }); + `, + { eval: true }, + ); + + const { port1, port2 } = new workerThreads.MessageChannel(); + + const done = Promise.withResolvers(); + + port1.on("message", (msg) => { + assertEquals(msg.type, "done"); + assertEquals(msg.got, "c"); + worker.unref(); + port1.close(); + done.resolve(true); + }); + worker.on("message", (msg) => { + assertEquals(msg, "ready"); + port1.postMessage("a"); + port1.postMessage("b"); + port1.postMessage("c"); + }); + worker.postMessage({ port: port2 }, [port2]); + + const timeout = setTimeout(() => { + fail("Test timed out"); + }, 20_000); + try { + const result = await done.promise; + assertEquals(result, true); + } finally { + clearTimeout(timeout); + } + }, +}); + +Deno.test({ + name: "[node/worker_threads] npm:piscina wait loop hang regression", + async fn() { + const worker = new workerThreads.Worker( + ` + import { assert, assertEquals } from "@std/assert"; + import { parentPort, receiveMessageOnPort } from "node:worker_threads"; + + assert(parentPort !== null); + + let currentTasks = 0; + let lastSeen = 0; + + parentPort.on("message", (msg) => { + (async () => { + assert(typeof msg === "object" && msg !== null); + assert(msg.buf !== undefined); + assert(msg.port !== undefined); + const { buf, port } = msg; + port.postMessage("ready"); + port.on("message", (msg) => onMessage(msg, buf, port)); + atomicsWaitLoop(buf, port); + })(); + }); + + function onMessage(msg, buf, port) { + currentTasks++; + (async () => { + assert(msg.taskName !== undefined); + port.postMessage({ type: "response", taskName: msg.taskName }); + currentTasks--; + atomicsWaitLoop(buf, port); + })(); + } + + function atomicsWaitLoop(buf, port) { + while (currentTasks === 0) { + Atomics.wait(buf, 0, lastSeen); + lastSeen = Atomics.load(buf, 0); + let task; + while ((task = receiveMessageOnPort(port)) !== undefined) { + onMessage(task.message, buf, port); + } + } + } + `, + { eval: true }, + ); + + const sab = new SharedArrayBuffer(4); + const buf = new Int32Array(sab); + const { port1, port2 } = new workerThreads.MessageChannel(); + + const done = Promise.withResolvers(); + + port1.unref(); + + worker.postMessage({ + type: "init", + buf, + port: port2, + }, [port2]); + + let count = 0; + port1.on("message", (msg) => { + if (count++ === 0) { + assertEquals(msg, "ready"); + } else { + assertEquals(msg.type, "response"); + port1.close(); + done.resolve(true); + } + }); + + port1.postMessage({ + taskName: "doThing", + }); + + Atomics.add(buf, 0, 1); + Atomics.notify(buf, 0, 1); + + worker.unref(); + + const result = await done.promise; + assertEquals(result, true); + }, +}); diff --git a/tests/unit_node/zlib_test.ts b/tests/unit_node/zlib_test.ts index 8bce5ce7d19189..de2d2450d1cc20 100644 --- a/tests/unit_node/zlib_test.ts +++ b/tests/unit_node/zlib_test.ts @@ -7,9 +7,11 @@ import { brotliCompressSync, brotliDecompressSync, constants, + crc32, createBrotliCompress, createBrotliDecompress, createDeflate, + gzip, gzipSync, unzipSync, } from "node:zlib"; @@ -210,3 +212,21 @@ Deno.test("createBrotliCompress params", async () => { ); assertEquals(output.length, input.length); }); + +Deno.test("gzip() and gzipSync() accept ArrayBuffer", async () => { + const deffered = Promise.withResolvers(); + const buf = new ArrayBuffer(0); + let output: Buffer; + gzip(buf, (_err, data) => { + output = data; + deffered.resolve(); + }); + await deffered.promise; + assert(output! instanceof Buffer); + const outputSync = gzipSync(buf); + assert(outputSync instanceof Buffer); +}); + +Deno.test("crc32()", () => { + assertEquals(crc32("hello world"), 222957957); +}); diff --git a/tests/util/server/Cargo.toml b/tests/util/server/Cargo.toml index a321501b85fdf6..efc81da17c55f5 100644 --- a/tests/util/server/Cargo.toml +++ b/tests/util/server/Cargo.toml @@ -21,6 +21,7 @@ bytes.workspace = true console_static_text.workspace = true deno_unsync = "0" denokv_proto.workspace = true +faster-hex.workspace = true fastwebsockets.workspace = true flate2 = { workspace = true, features = ["default"] } futures.workspace = true @@ -35,7 +36,7 @@ lazy-regex.workspace = true libc.workspace = true lsp-types.workspace = true monch.workspace = true -nix.workspace = true +nix = { workspace = true, features = ["fs", "term", "signal"] } once_cell.workspace = true os_pipe.workspace = true parking_lot.workspace = true diff --git a/tests/util/server/src/builders.rs b/tests/util/server/src/builders.rs index 4a4b6a76136632..4a1510ce4c1017 100644 --- a/tests/util/server/src/builders.rs +++ b/tests/util/server/src/builders.rs @@ -28,6 +28,7 @@ use crate::fs::PathRef; use crate::http_server; use crate::jsr_registry_unset_url; use crate::lsp::LspClientBuilder; +use crate::nodejs_org_mirror_unset_url; use crate::npm_registry_unset_url; use crate::pty::Pty; use crate::strip_ansi_codes; @@ -78,6 +79,7 @@ impl DiagnosticLogger { logger.write_all(text.as_ref().as_bytes()).unwrap(); logger.write_all(b"\n").unwrap(); } + #[allow(clippy::print_stderr)] None => eprintln!("{}", text.as_ref()), } } @@ -842,6 +844,12 @@ impl TestCommandBuilder { if !envs.contains_key("JSR_URL") { envs.insert("JSR_URL".to_string(), jsr_registry_unset_url()); } + if !envs.contains_key("NODEJS_ORG_MIRROR") { + envs.insert( + "NODEJS_ORG_MIRROR".to_string(), + nodejs_org_mirror_unset_url(), + ); + } for key in &self.envs_remove { envs.remove(key); } diff --git a/tests/util/server/src/fs.rs b/tests/util/server/src/fs.rs index 47d0d61fa1236a..7feb0799ae45b9 100644 --- a/tests/util/server/src/fs.rs +++ b/tests/util/server/src/fs.rs @@ -285,7 +285,10 @@ impl PathRef { #[track_caller] pub fn assert_matches_file(&self, wildcard_file: impl AsRef) -> &Self { let wildcard_file = testdata_path().join(wildcard_file); - println!("output path {}", wildcard_file); + #[allow(clippy::print_stdout)] + { + println!("output path {}", wildcard_file); + } let expected_text = wildcard_file.read_to_string(); self.assert_matches_text(&expected_text) } diff --git a/tests/util/server/src/lib.rs b/tests/util/server/src/lib.rs index 88e8287e0887ab..953896cffda570 100644 --- a/tests/util/server/src/lib.rs +++ b/tests/util/server/src/lib.rs @@ -1,8 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -#![allow(clippy::print_stdout)] -#![allow(clippy::print_stderr)] - use std::collections::HashMap; use std::env; use std::io::Write; @@ -55,6 +52,7 @@ static GUARD: Lazy> = Lazy::new(Default::default); pub fn env_vars_for_npm_tests() -> Vec<(String, String)> { vec![ ("NPM_CONFIG_REGISTRY".to_string(), npm_registry_url()), + ("NODEJS_ORG_MIRROR".to_string(), nodejs_org_mirror_url()), ("NO_COLOR".to_string(), "1".to_string()), ] } @@ -133,6 +131,7 @@ pub fn env_vars_for_jsr_npm_tests() -> Vec<(String, String)> { ), ("DISABLE_JSR_PROVENANCE".to_string(), "true".to_string()), ("NO_COLOR".to_string(), "1".to_string()), + ("NODEJS_ORG_MIRROR".to_string(), nodejs_org_mirror_url()), ] } @@ -178,27 +177,41 @@ pub fn deno_config_path() -> PathRef { /// Test server registry url. pub fn npm_registry_url() -> String { - "http://localhost:4260/".to_string() + format!("http://localhost:{}/", servers::PUBLIC_NPM_REGISTRY_PORT) } pub fn npm_registry_unset_url() -> String { "http://NPM_CONFIG_REGISTRY.is.unset".to_string() } +pub fn nodejs_org_mirror_url() -> String { + format!( + "http://127.0.0.1:{}/", + servers::NODEJS_ORG_MIRROR_SERVER_PORT + ) +} + +pub fn nodejs_org_mirror_unset_url() -> String { + "http://NODEJS_ORG_MIRROR.is.unset".to_string() +} + pub fn jsr_registry_url() -> String { - "http://127.0.0.1:4250/".to_string() + format!("http://127.0.0.1:{}/", servers::JSR_REGISTRY_SERVER_PORT) } pub fn rekor_url() -> String { - "http://127.0.0.1:4251".to_string() + format!("http://127.0.0.1:{}", servers::PROVENANCE_MOCK_SERVER_PORT) } pub fn fulcio_url() -> String { - "http://127.0.0.1:4251".to_string() + format!("http://127.0.0.1:{}", servers::PROVENANCE_MOCK_SERVER_PORT) } pub fn gha_token_url() -> String { - "http://127.0.0.1:4251/gha_oidc?test=true".to_string() + format!( + "http://127.0.0.1:{}/gha_oidc?test=true", + servers::PROVENANCE_MOCK_SERVER_PORT + ) } pub fn jsr_registry_unset_url() -> String { @@ -302,12 +315,15 @@ async fn get_tcp_listener_stream( .collect::>(); // Eye catcher for HttpServerCount - println!("ready: {name} on {:?}", addresses); + #[allow(clippy::print_stdout)] + { + println!("ready: {name} on {:?}", addresses); + } futures::stream::select_all(listeners) } -pub const TEST_SERVERS_COUNT: usize = 30; +pub const TEST_SERVERS_COUNT: usize = 33; #[derive(Default)] struct HttpServerCount { @@ -345,7 +361,10 @@ struct HttpServerStarter { impl Default for HttpServerStarter { fn default() -> Self { - println!("test_server starting..."); + #[allow(clippy::print_stdout)] + { + println!("test_server starting..."); + } let mut test_server = Command::new(test_server_path()) .current_dir(testdata_path()) .stdout(Stdio::piped()) @@ -479,6 +498,7 @@ pub fn run_collect( } = prog.wait_with_output().expect("failed to wait on child"); let stdout = String::from_utf8(stdout).unwrap(); let stderr = String::from_utf8(stderr).unwrap(); + #[allow(clippy::print_stderr)] if expect_success != status.success() { eprintln!("stdout: <<<{stdout}>>>"); eprintln!("stderr: <<<{stderr}>>>"); @@ -539,6 +559,7 @@ pub fn run_and_collect_output_with_args( } = deno.wait_with_output().expect("failed to wait on child"); let stdout = String::from_utf8(stdout).unwrap(); let stderr = String::from_utf8(stderr).unwrap(); + #[allow(clippy::print_stderr)] if expect_success != status.success() { eprintln!("stdout: <<<{stdout}>>>"); eprintln!("stderr: <<<{stderr}>>>"); @@ -560,9 +581,11 @@ pub fn deno_cmd_with_deno_dir(deno_dir: &TempDir) -> TestCommandBuilder { TestCommandBuilder::new(deno_dir.clone()) .env("DENO_DIR", deno_dir.path()) .env("NPM_CONFIG_REGISTRY", npm_registry_unset_url()) + .env("NODEJS_ORG_MIRROR", nodejs_org_mirror_unset_url()) .env("JSR_URL", jsr_registry_unset_url()) } +#[allow(clippy::print_stdout)] pub fn run_powershell_script_file( script_file_path: &str, args: Vec<&str>, @@ -654,6 +677,7 @@ impl<'a> CheckOutputIntegrationTest<'a> { } pub fn wildcard_match(pattern: &str, text: &str) -> bool { + #[allow(clippy::print_stderr)] match wildcard_match_detailed(pattern, text) { WildcardMatchResult::Success => true, WildcardMatchResult::Fail(debug_output) => { @@ -895,6 +919,11 @@ pub fn wildcard_match_detailed( if was_last_wildcard || was_last_wildline || current_text.is_empty() { WildcardMatchResult::Success + } else if current_text == "\n" { + WildcardMatchResult::Fail( + "\n!!!! PROBLEM: Missing final newline at end of expected output !!!!" + .to_string(), + ) } else { output_lines.push("==== HAD TEXT AT END OF FILE ====".to_string()); output_lines.push(colors::red(annotate_whitespace(current_text))); diff --git a/tests/util/server/src/lsp.rs b/tests/util/server/src/lsp.rs index ffe72b88af1fab..d34deb21619a37 100644 --- a/tests/util/server/src/lsp.rs +++ b/tests/util/server/src/lsp.rs @@ -157,6 +157,7 @@ impl LspStdoutReader { self.pending_messages.0.lock().len() } + #[allow(clippy::print_stderr)] pub fn output_pending_messages(&self) { let messages = self.pending_messages.0.lock(); eprintln!("{:?}", messages); @@ -308,34 +309,6 @@ impl InitializeParamsBuilder { self } - pub fn enable_inlay_hints(&mut self) -> &mut Self { - let options = self.initialization_options_mut(); - options.insert( - "inlayHints".to_string(), - json!({ - "parameterNames": { - "enabled": "all" - }, - "parameterTypes": { - "enabled": true - }, - "variableTypes": { - "enabled": true - }, - "propertyDeclarationTypes": { - "enabled": true - }, - "functionLikeReturnTypes": { - "enabled": true - }, - "enumMemberValues": { - "enabled": true - } - }), - ); - self - } - pub fn disable_testing_api(&mut self) -> &mut Self { let obj = self .params @@ -601,6 +574,7 @@ impl LspClientBuilder { for line in stderr.lines() { match line { Ok(line) => { + #[allow(clippy::print_stderr)] if print_stderr { eprintln!("{}", line); } @@ -615,7 +589,10 @@ impl LspClientBuilder { continue; } Err(err) => { - eprintln!("failed to parse perf record: {:#}", err); + #[allow(clippy::print_stderr)] + { + eprintln!("failed to parse perf record: {:#}", err); + } } } } @@ -810,11 +787,14 @@ impl LspClient { std::thread::sleep(Duration::from_millis(20)); } - eprintln!("==== STDERR OUTPUT ===="); - for line in found_lines { - eprintln!("{}", line) + #[allow(clippy::print_stderr)] + { + eprintln!("==== STDERR OUTPUT ===="); + for line in found_lines { + eprintln!("{}", line) + } + eprintln!("== END STDERR OUTPUT =="); } - eprintln!("== END STDERR OUTPUT =="); panic!("Timed out waiting on condition.") } diff --git a/tests/util/server/src/macros.rs b/tests/util/server/src/macros.rs index fdbb977e9f5552..e076583f19b306 100644 --- a/tests/util/server/src/macros.rs +++ b/tests/util/server/src/macros.rs @@ -33,6 +33,7 @@ macro_rules! timeout { use std::io::Write; eprintln!("Test {function} timed out after {timeout} seconds, aborting"); _ = std::io::stderr().flush(); + #[allow(clippy::disallowed_methods)] ::std::process::exit(1); } }); diff --git a/tests/util/server/src/npm.rs b/tests/util/server/src/npm.rs index f1c341738adba6..081989ddb5fed1 100644 --- a/tests/util/server/src/npm.rs +++ b/tests/util/server/src/npm.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::fs; +use std::path::Path; use anyhow::Context; use anyhow::Result; @@ -18,6 +19,7 @@ use crate::PathRef; pub const DENOTEST_SCOPE_NAME: &str = "@denotest"; pub const DENOTEST2_SCOPE_NAME: &str = "@denotest2"; +pub const DENOTEST3_SCOPE_NAME: &str = "@denotest3"; pub static PUBLIC_TEST_NPM_REGISTRY: Lazy = Lazy::new(|| { TestNpmRegistry::new( @@ -54,6 +56,18 @@ pub static PRIVATE_TEST_NPM_REGISTRY_2: Lazy = ) }); +pub static PRIVATE_TEST_NPM_REGISTRY_3: Lazy = + Lazy::new(|| { + TestNpmRegistry::new( + NpmRegistryKind::Private, + &format!( + "http://localhost:{}", + crate::servers::PRIVATE_NPM_REGISTRY_3_PORT + ), + "npm-private3", + ) + }); + pub enum NpmRegistryKind { Public, Private, @@ -161,10 +175,75 @@ impl TestNpmRegistry { return Some((DENOTEST2_SCOPE_NAME, package_name_with_path)); } + let prefix1 = format!("/{}/", DENOTEST3_SCOPE_NAME); + let prefix2 = format!("/{}%2f", DENOTEST3_SCOPE_NAME); + + let maybe_package_name_with_path = uri_path + .strip_prefix(&prefix1) + .or_else(|| uri_path.strip_prefix(&prefix2)); + + if let Some(package_name_with_path) = maybe_package_name_with_path { + return Some((DENOTEST3_SCOPE_NAME, package_name_with_path)); + } + None } } +// NOTE: extracted out partially from the `tar` crate, all credits to the original authors +fn append_dir_all( + builder: &mut tar::Builder, + path: &Path, + src_path: &Path, +) -> Result<()> { + builder.follow_symlinks(true); + let mode = tar::HeaderMode::Deterministic; + builder.mode(mode); + let mut stack = vec![(src_path.to_path_buf(), true, false)]; + let mut entries = Vec::new(); + while let Some((src, is_dir, is_symlink)) = stack.pop() { + let dest = path.join(src.strip_prefix(src_path).unwrap()); + // In case of a symlink pointing to a directory, is_dir is false, but src.is_dir() will return true + if is_dir || (is_symlink && src.is_dir()) { + for entry in fs::read_dir(&src)? { + let entry = entry?; + let file_type = entry.file_type()?; + stack.push((entry.path(), file_type.is_dir(), file_type.is_symlink())); + } + if dest != Path::new("") { + entries.push((src, dest)); + } + } else { + entries.push((src, dest)); + } + } + entries.sort_by(|(_, a), (_, b)| a.cmp(b)); + for (src, dest) in entries { + let mut header = tar::Header::new_gnu(); + let metadata = src.metadata().with_context(|| { + format!("trying to get metadata for {}", src.display()) + })?; + header.set_metadata_in_mode(&metadata, mode); + // this is what `tar` sets the mtime to on unix in deterministic mode, on windows it uses a different + // value, which causes the tarball to have a different hash on windows. force it to be the same + // to ensure the same output on all platforms + header.set_mtime(1153704088); + + let data = if src.is_file() { + Box::new( + fs::File::open(&src) + .with_context(|| format!("trying to open file {}", src.display()))?, + ) as Box + } else { + Box::new(std::io::empty()) as Box + }; + builder + .append_data(&mut header, dest, data) + .with_context(|| "appending data")?; + } + Ok(()) +} + fn get_npm_package( registry_hostname: &str, local_path: &str, @@ -204,11 +283,14 @@ fn get_npm_package( GzEncoder::new(&mut tarball_bytes, Compression::default()); { let mut builder = Builder::new(&mut encoder); - builder - .append_dir_all("package", &version_folder) - .with_context(|| { - format!("Error adding tarball for directory: {}", version_folder) - })?; + append_dir_all( + &mut builder, + Path::new("package"), + version_folder.as_path(), + ) + .with_context(|| { + format!("Error adding tarball for directory {}", version_folder,) + })?; builder.finish()?; } encoder.finish()?; diff --git a/tests/util/server/src/pty.rs b/tests/util/server/src/pty.rs index 5d8049fee97af6..07659262cf4e4c 100644 --- a/tests/util/server/src/pty.rs +++ b/tests/util/server/src/pty.rs @@ -61,7 +61,10 @@ impl Pty { if is_windows && *IS_CI { // the pty tests don't really start up on the windows CI for some reason // so ignore them for now - eprintln!("Ignoring windows CI."); + #[allow(clippy::print_stderr)] + { + eprintln!("Ignoring windows CI."); + } false } else { true @@ -250,11 +253,14 @@ impl Pty { } let text = self.next_text(); - eprintln!( - "------ Start Full Text ------\n{:?}\n------- End Full Text -------", - String::from_utf8_lossy(&self.read_bytes) - ); - eprintln!("Next text: {:?}", text); + #[allow(clippy::print_stderr)] + { + eprintln!( + "------ Start Full Text ------\n{:?}\n------- End Full Text -------", + String::from_utf8_lossy(&self.read_bytes) + ); + eprintln!("Next text: {:?}", text); + } false } @@ -297,10 +303,12 @@ fn setup_pty(fd: i32) { use nix::sys::termios::tcsetattr; use nix::sys::termios::SetArg; - let mut term = tcgetattr(fd).unwrap(); + // SAFETY: Nix crate requires value to implement the AsFd trait + let as_fd = unsafe { std::os::fd::BorrowedFd::borrow_raw(fd) }; + let mut term = tcgetattr(as_fd).unwrap(); // disable cooked mode term.local_flags.remove(termios::LocalFlags::ICANON); - tcsetattr(fd, SetArg::TCSANOW, &term).unwrap(); + tcsetattr(as_fd, SetArg::TCSANOW, &term).unwrap(); // turn on non-blocking mode so we get timeouts let flags = fcntl(fd, FcntlArg::F_GETFL).unwrap(); diff --git a/tests/util/server/src/servers/hyper_utils.rs b/tests/util/server/src/servers/hyper_utils.rs index c2db7ea6688c5a..8e01151ed494bb 100644 --- a/tests/util/server/src/servers/hyper_utils.rs +++ b/tests/util/server/src/servers/hyper_utils.rs @@ -42,7 +42,10 @@ where let fut: Pin>>> = async move { let listener = TcpListener::bind(options.addr).await?; - println!("ready: {}", options.addr); + #[allow(clippy::print_stdout)] + { + println!("ready: {}", options.addr); + } loop { let (stream, _) = listener.accept().await?; let io = TokioIo::new(stream); @@ -58,6 +61,7 @@ where if let Err(e) = fut.await { let err_str = e.to_string(); + #[allow(clippy::print_stderr)] if !err_str.contains("early eof") { eprintln!("{}: {:?}", options.error_msg, e); } @@ -89,6 +93,7 @@ pub async fn run_server_with_acceptor<'a, A, F, S>( if let Err(e) = fut.await { let err_str = e.to_string(); + #[allow(clippy::print_stderr)] if !err_str.contains("early eof") { eprintln!("{}: {:?}", error_msg, e); } @@ -135,6 +140,7 @@ async fn hyper_serve_connection( if let Err(e) = result { let err_str = e.to_string(); + #[allow(clippy::print_stderr)] if !err_str.contains("early eof") { eprintln!("{}: {:?}", error_msg, e); } diff --git a/tests/util/server/src/servers/mod.rs b/tests/util/server/src/servers/mod.rs index 3e18aafce49590..0b1d99aeb92751 100644 --- a/tests/util/server/src/servers/mod.rs +++ b/tests/util/server/src/servers/mod.rs @@ -39,6 +39,7 @@ use tokio::net::TcpStream; mod grpc; mod hyper_utils; mod jsr_registry; +mod nodejs_org_mirror; mod npm_registry; mod ws; @@ -86,11 +87,13 @@ const WS_CLOSE_PORT: u16 = 4244; const WS_PING_PORT: u16 = 4245; const H2_GRPC_PORT: u16 = 4246; const H2S_GRPC_PORT: u16 = 4247; -const JSR_REGISTRY_SERVER_PORT: u16 = 4250; -const PROVENANCE_MOCK_SERVER_PORT: u16 = 4251; +pub(crate) const JSR_REGISTRY_SERVER_PORT: u16 = 4250; +pub(crate) const PROVENANCE_MOCK_SERVER_PORT: u16 = 4251; +pub(crate) const NODEJS_ORG_MIRROR_SERVER_PORT: u16 = 4252; pub(crate) const PUBLIC_NPM_REGISTRY_PORT: u16 = 4260; pub(crate) const PRIVATE_NPM_REGISTRY_1_PORT: u16 = 4261; pub(crate) const PRIVATE_NPM_REGISTRY_2_PORT: u16 = 4262; +pub(crate) const PRIVATE_NPM_REGISTRY_3_PORT: u16 = 4263; // Use the single-threaded scheduler. The hyper server is used as a point of // comparison for the (single-threaded!) benchmarks in cli/bench. We're not @@ -143,6 +146,12 @@ pub async fn run_all_servers() { npm_registry::private_npm_registry1(PRIVATE_NPM_REGISTRY_1_PORT); let private_npm_registry_2_server_futs = npm_registry::private_npm_registry2(PRIVATE_NPM_REGISTRY_2_PORT); + let private_npm_registry_3_server_futs = + npm_registry::private_npm_registry3(PRIVATE_NPM_REGISTRY_3_PORT); + + // for serving node header files to node-gyp in tests + let node_js_mirror_server_fut = + nodejs_org_mirror::nodejs_org_mirror(NODEJS_ORG_MIRROR_SERVER_PORT); let mut futures = vec![ redirect_server_fut.boxed_local(), @@ -169,10 +178,12 @@ pub async fn run_all_servers() { h2_grpc_server_fut.boxed_local(), registry_server_fut.boxed_local(), provenance_mock_server_fut.boxed_local(), + node_js_mirror_server_fut.boxed_local(), ]; futures.extend(npm_registry_server_futs); futures.extend(private_npm_registry_1_server_futs); futures.extend(private_npm_registry_2_server_futs); + futures.extend(private_npm_registry_3_server_futs); assert_eq!(futures.len(), TEST_SERVERS_COUNT); @@ -194,7 +205,6 @@ fn json_body(value: serde_json::Value) -> UnsyncBoxBody { /// Benchmark server that just serves "hello world" responses. async fn hyper_hello(port: u16) { - println!("hyper hello"); let addr = SocketAddr::from(([127, 0, 0, 1], port)); let handler = move |_: Request| async move { Ok::<_, anyhow::Error>(Response::new(UnsyncBoxBody::new( @@ -338,7 +348,10 @@ async fn get_tcp_listener_stream( .collect::>(); // Eye catcher for HttpServerCount - println!("ready: {name} on {:?}", addresses); + #[allow(clippy::print_stdout)] + { + println!("ready: {name} on {:?}", addresses); + } futures::stream::select_all(listeners) } @@ -354,7 +367,10 @@ async fn run_tls_client_auth_server(port: u16) { while let Some(Ok(mut tls_stream)) = tls.next().await { tokio::spawn(async move { let Ok(handshake) = tls_stream.handshake().await else { - eprintln!("Failed to handshake"); + #[allow(clippy::print_stderr)] + { + eprintln!("Failed to handshake"); + } return; }; // We only need to check for the presence of client certificates @@ -401,7 +417,6 @@ async fn absolute_redirect( .collect(); if let Some(url) = query_params.get("redirect_to") { - println!("URL: {url:?}"); let redirect = redirect_resp(url.to_owned()); return Ok(redirect); } @@ -409,7 +424,6 @@ async fn absolute_redirect( if path.starts_with("/REDIRECT") { let url = &req.uri().path()[9..]; - println!("URL: {url:?}"); let redirect = redirect_resp(url.to_string()); return Ok(redirect); } @@ -793,17 +807,17 @@ async fn main_server( (_, "/jsx/jsx-runtime") | (_, "/jsx/jsx-dev-runtime") => { let mut res = Response::new(string_body( r#"export function jsx( - _type, - _props, - _key, - _source, - _self, - ) {} - export const jsxs = jsx; - export const jsxDEV = jsx; - export const Fragment = Symbol("Fragment"); - console.log("imported", import.meta.url); - "#, + _type, + _props, + _key, + _source, + _self, +) {} +export const jsxs = jsx; +export const jsxDEV = jsx; +export const Fragment = Symbol("Fragment"); +console.log("imported", import.meta.url); +"#, )); res.headers_mut().insert( "Content-type", @@ -1353,6 +1367,7 @@ async fn wrap_client_auth_https_server(port: u16) { // here. Rusttls ensures that they are valid and signed by the CA. match handshake.has_peer_certificates { true => { yield Ok(tls); }, + #[allow(clippy::print_stderr)] false => { eprintln!("https_client_auth: no valid client certificate"); }, }; } diff --git a/tests/util/server/src/servers/nodejs_org_mirror.rs b/tests/util/server/src/servers/nodejs_org_mirror.rs new file mode 100644 index 00000000000000..521e79d3c458fa --- /dev/null +++ b/tests/util/server/src/servers/nodejs_org_mirror.rs @@ -0,0 +1,245 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +//! Server for NodeJS header tarballs, used by `node-gyp` in tests to download headers +//! +//! Loads from `testdata/assets`, if we update our node version in `process.versions` we'll need to +//! update the header tarball there. + +#![allow(clippy::print_stderr)] + +use std::collections::HashMap; +use std::convert::Infallible; +use std::net::SocketAddr; +use std::sync::LazyLock; + +use bytes::Bytes; +use http::Response; +use http::StatusCode; +use http_body_util::combinators::UnsyncBoxBody; +use http_body_util::Full; +use parking_lot::Mutex; + +use crate::servers::hyper_utils::run_server; +use crate::servers::hyper_utils::ServerKind; +use crate::servers::hyper_utils::ServerOptions; +use crate::servers::string_body; +use crate::testdata_path; +use crate::PathRef; + +/// a little helper extension trait to log errors but convert to option +trait OkWarn { + fn ok_warn(self) -> Option; +} + +impl OkWarn for Result +where + E: std::fmt::Display, +{ + fn ok_warn(self) -> Option { + self + .inspect_err(|err| { + eprintln!( + "test_server warning: error occurred in nodejs_org_mirror.rs: {err}" + ) + }) + .ok() + } +} + +pub static NODEJS_MIRROR: LazyLock = + LazyLock::new(NodeJsMirror::default); + +#[derive(Default)] +pub struct NodeJsMirror { + cache: Mutex>, + checksum_cache: Mutex>, +} + +fn asset_file_path(file: &str) -> PathRef { + testdata_path().join("assets").join("node-gyp").join(file) +} + +impl NodeJsMirror { + pub fn get_header_bytes(&self, file: &str) -> Option { + let mut cache = self.cache.lock(); + let entry = cache.entry(file.to_owned()); + match entry { + std::collections::hash_map::Entry::Occupied(occupied) => { + Some(occupied.get().clone()) + } + std::collections::hash_map::Entry::Vacant(vacant) => { + let contents = asset_file_path(file); + let contents = contents + .read_to_bytes_if_exists() + .ok_warn() + .map(Bytes::from)?; + vacant.insert(contents.clone()); + Some(contents) + } + } + } + + fn get_checksum(&self, file: &str, bytes: Bytes) -> String { + use sha2::Digest; + if let Some(checksum) = self.checksum_cache.lock().get(file).cloned() { + return checksum; + } + let mut hasher = sha2::Sha256::new(); + hasher.update(&bytes); + let checksum = faster_hex::hex_string(hasher.finalize().as_ref()); + self + .checksum_cache + .lock() + .insert(file.to_owned(), checksum.clone()); + checksum + } + + pub fn get_checksum_file(&self, version: &str) -> Option { + let mut entries = Vec::with_capacity(2); + + let header_file = header_tar_name(version); + let header_bytes = self.get_header_bytes(&header_file)?; + let header_checksum = self.get_checksum(&header_file, header_bytes); + entries.push((header_file, header_checksum)); + + if cfg!(windows) { + if !cfg!(target_arch = "x86_64") { + panic!("unsupported target arch on windows, only support x86_64"); + } + let Some(bytes) = self.get_node_lib_bytes(version, "win-x64") else { + eprintln!("test server failed to get node lib"); + return None; + }; + { + let file = format!("{version}/win-x64/node.lib"); + let checksum = self.get_checksum(&file, bytes); + let filename_for_checksum = + file.trim_start_matches(&format!("{version}/")); + entries.push((filename_for_checksum.to_owned(), checksum)); + } + } + + Some( + entries + .into_iter() + .map(|(file, checksum)| format!("{checksum} {file}")) + .collect::>() + .join("\n"), + ) + } + + pub fn get_node_lib_bytes( + &self, + version: &str, + platform: &str, + ) -> Option { + let mut cache = self.cache.lock(); + let file_name = format!("{version}/{platform}/node.lib"); + let entry = cache.entry(file_name); + match entry { + std::collections::hash_map::Entry::Occupied(occupied) => { + Some(occupied.get().clone()) + } + std::collections::hash_map::Entry::Vacant(vacant) => { + let tarball_filename = + format!("{version}__{platform}__node.lib.tar.gz"); + let contents = asset_file_path(&tarball_filename); + let contents = contents.read_to_bytes_if_exists().ok_warn()?; + let extracted = Bytes::from(extract_tarball(&contents)?); + vacant.insert(extracted.clone()); + Some(extracted) + } + } + } +} + +fn header_tar_name(version: &str) -> String { + format!("node-{version}-headers.tar.gz") +} + +fn extract_tarball(compressed: &[u8]) -> Option> { + let mut out = Vec::with_capacity(compressed.len()); + let decoder = flate2::read::GzDecoder::new(compressed); + let mut archive = tar::Archive::new(decoder); + for file in archive.entries().ok_warn()? { + let mut file = file.ok_warn()?; + + std::io::copy(&mut file, &mut out).ok_warn()?; + } + Some(out) +} + +/// Server for node JS header tarballs, used by `node-gyp` in tests +pub async fn nodejs_org_mirror(port: u16) { + let addr = SocketAddr::from(([127, 0, 0, 1], port)); + + run_server( + ServerOptions { + addr, + error_msg: "nodejs mirror server error", + kind: ServerKind::Auto, + }, + |req| async move { + let path = req.uri().path(); + if path.contains("-headers.tar.gz") + || path.contains("SHASUMS256.txt") + || path.contains("node.lib") + { + let mut parts = path.split('/'); + let _ = parts.next(); // empty + let Some(version) = parts.next() else { + return not_found(format!("missing node version in path: {path}")); + }; + let Some(file) = parts.next() else { + return not_found(format!("missing file version in path: {path}")); + }; + if file == "SHASUMS256.txt" { + let Some(checksum_file) = NODEJS_MIRROR.get_checksum_file(version) + else { + return not_found(format!("failed to get header checksum: {path}")); + }; + return Ok(Response::new(string_body(&checksum_file))); + } else if !file.contains("headers") { + let platform = file; + let Some(file) = parts.next() else { + return not_found("expected file"); + }; + if file != "node.lib" { + return not_found(format!( + "unexpected file name, expected node.lib, got: {file}" + )); + } + let Some(bytes) = NODEJS_MIRROR.get_node_lib_bytes(version, platform) + else { + return not_found("expected node lib bytes"); + }; + + return Ok(Response::new(UnsyncBoxBody::new(Full::new(bytes)))); + } + + let Some(bytes) = NODEJS_MIRROR.get_header_bytes(file) else { + return not_found(format!( + "couldn't find headers for version {version}, missing file: {file}" + )); + }; + Ok(Response::new(UnsyncBoxBody::new(Full::new(bytes)))) + } else { + not_found(format!("unexpected request path: {path}")) + } + }, + ) + .await +} + +fn not_found( + msg: impl AsRef, +) -> Result>, anyhow::Error> { + let msg = msg.as_ref(); + eprintln!( + "test_server warning: error likely occurred in nodejs_org_mirror.rs: {msg}" + ); + Response::builder() + .status(StatusCode::NOT_FOUND) + .body(string_body(msg)) + .map_err(|e| e.into()) +} diff --git a/tests/util/server/src/servers/npm_registry.rs b/tests/util/server/src/servers/npm_registry.rs index acbd9cab48af8f..4ada468fac4577 100644 --- a/tests/util/server/src/servers/npm_registry.rs +++ b/tests/util/server/src/servers/npm_registry.rs @@ -56,6 +56,14 @@ pub fn private_npm_registry2(port: u16) -> Vec> { ) } +pub fn private_npm_registry3(port: u16) -> Vec> { + run_npm_server( + port, + "npm private registry server error", + private_npm_registry3_handler, + ) +} + fn run_npm_server( port: u16, error_msg: &'static str, @@ -141,6 +149,13 @@ async fn private_npm_registry2_handler( handle_req_for_registry(req, &npm::PRIVATE_TEST_NPM_REGISTRY_2).await } +async fn private_npm_registry3_handler( + req: Request, +) -> Result>, anyhow::Error> { + // No auth for this registry + handle_req_for_registry(req, &npm::PRIVATE_TEST_NPM_REGISTRY_3).await +} + async fn handle_req_for_registry( req: Request, test_npm_registry: &npm::TestNpmRegistry, diff --git a/tests/util/server/src/servers/ws.rs b/tests/util/server/src/servers/ws.rs index 815119b6a13dcd..dd4efbf659809e 100644 --- a/tests/util/server/src/servers/ws.rs +++ b/tests/util/server/src/servers/ws.rs @@ -76,6 +76,7 @@ pub async fn run_wss2_server(port: u16) { let server: Handshake<_, Bytes> = h2.handshake(tls); let mut server = match server.await { Ok(server) => server, + #[allow(clippy::print_stdout)] Err(e) => { println!("Failed to handshake h2: {e:?}"); return; @@ -87,6 +88,7 @@ pub async fn run_wss2_server(port: u16) { }; let (recv, send) = match conn { Ok(conn) => conn, + #[allow(clippy::print_stdout)] Err(e) => { println!("Failed to accept a connection: {e:?}"); break; @@ -137,6 +139,7 @@ where .map_err(|e| anyhow!("Error upgrading websocket connection: {}", e)) .unwrap(); + #[allow(clippy::print_stderr)] if let Err(e) = handler(ws).await { eprintln!("Error in websocket connection: {}", e); } @@ -152,6 +155,7 @@ where .serve_connection(io, service) .with_upgrades(); + #[allow(clippy::print_stderr)] if let Err(e) = conn.await { eprintln!("websocket server error: {e:?}"); } @@ -162,16 +166,19 @@ async fn handle_wss_stream( recv: Request, mut send: SendResponse, ) -> Result<(), h2::Error> { + #[allow(clippy::print_stderr)] if recv.method() != Method::CONNECT { eprintln!("wss2: refusing non-CONNECT stream"); send.send_reset(Reason::REFUSED_STREAM); return Ok(()); } + #[allow(clippy::print_stderr)] let Some(protocol) = recv.extensions().get::() else { eprintln!("wss2: refusing no-:protocol stream"); send.send_reset(Reason::REFUSED_STREAM); return Ok(()); }; + #[allow(clippy::print_stderr)] if protocol.as_str() != "websocket" && protocol.as_str() != "WebSocket" { eprintln!("wss2: refusing non-websocket stream"); send.send_reset(Reason::REFUSED_STREAM); diff --git a/tests/wpt/runner/expectation.json b/tests/wpt/runner/expectation.json index 1b31f26e5a98d8..5776fdb486b212 100644 --- a/tests/wpt/runner/expectation.json +++ b/tests/wpt/runner/expectation.json @@ -54,127 +54,65 @@ "pbkdf2.https.any.worker.html?7001-8000": true, "pbkdf2.https.any.worker.html?8001-last": true, "cfrg_curves_bits.https.any.html": [ - "X25519 key derivation checks for all-zero value result with a key of order 0", - "X25519 key derivation checks for all-zero value result with a key of order 1", - "X25519 key derivation checks for all-zero value result with a key of order 8", - "X25519 key derivation checks for all-zero value result with a key of order p-1 (order 2)", - "X25519 key derivation checks for all-zero value result with a key of order p (=0, order 4)", - "X25519 key derivation checks for all-zero value result with a key of order p+1 (=1, order 1)", "X448 key derivation checks for all-zero value result with a key of order 0", "X448 key derivation checks for all-zero value result with a key of order 1", "X448 key derivation checks for all-zero value result with a key of order p-1 (order 2)", "X448 key derivation checks for all-zero value result with a key of order p (=0, order 4)", "X448 key derivation checks for all-zero value result with a key of order p+1 (=1, order 1)", - "X25519 good parameters", - "X25519 mixed case parameters", - "X25519 short result", - "X25519 non-multiple of 8 bits", - "X25519 mismatched algorithms", - "X25519 no deriveBits usage for base key", "X448 good parameters", "X448 mixed case parameters", "X448 short result", "X448 non-multiple of 8 bits", "X448 mismatched algorithms", "X448 no deriveBits usage for base key", - "X448 base key is not a private key", "X448 public property value is a private key", "X448 public property value is a secret key", "X448 asking for too many bits" ], "cfrg_curves_bits.https.any.worker.html": [ - "X25519 key derivation checks for all-zero value result with a key of order 0", - "X25519 key derivation checks for all-zero value result with a key of order 1", - "X25519 key derivation checks for all-zero value result with a key of order 8", - "X25519 key derivation checks for all-zero value result with a key of order p-1 (order 2)", - "X25519 key derivation checks for all-zero value result with a key of order p (=0, order 4)", - "X25519 key derivation checks for all-zero value result with a key of order p+1 (=1, order 1)", "X448 key derivation checks for all-zero value result with a key of order 0", "X448 key derivation checks for all-zero value result with a key of order 1", "X448 key derivation checks for all-zero value result with a key of order p-1 (order 2)", "X448 key derivation checks for all-zero value result with a key of order p (=0, order 4)", "X448 key derivation checks for all-zero value result with a key of order p+1 (=1, order 1)", - "X25519 good parameters", - "X25519 mixed case parameters", - "X25519 short result", - "X25519 non-multiple of 8 bits", - "X25519 mismatched algorithms", - "X25519 no deriveBits usage for base key", "X448 good parameters", "X448 mixed case parameters", "X448 short result", "X448 non-multiple of 8 bits", "X448 mismatched algorithms", "X448 no deriveBits usage for base key", - "X448 base key is not a private key", "X448 public property value is a private key", "X448 public property value is a secret key", "X448 asking for too many bits" ], "cfrg_curves_keys.https.any.html": [ - "X25519 deriveBits checks for all-zero value result with a key of order 0", - "X25519 deriveBits checks for all-zero value result with a key of order 1", - "X25519 deriveBits checks for all-zero value result with a key of order 8", - "X25519 deriveBits checks for all-zero value result with a key of order p-1 (order 2)", - "X25519 deriveBits checks for all-zero value result with a key of order p (=0, order 4)", - "X25519 deriveBits checks for all-zero value result with a key of order p+1 (=1, order 1)", "X448 deriveBits checks for all-zero value result with a key of order 0", "X448 deriveBits checks for all-zero value result with a key of order 1", "X448 deriveBits checks for all-zero value result with a key of order p-1 (order 2)", "X448 deriveBits checks for all-zero value result with a key of order p (=0, order 4)", "X448 deriveBits checks for all-zero value result with a key of order p+1 (=1, order 1)", - "Key derivation using a X25519 generated keys.", - "Key derivation using a X448 generated keys.", - "X25519 good parameters", - "X25519 mixed case parameters", - "X25519 mismatched algorithms", "X448 good parameters", "X448 mixed case parameters", "X448 mismatched algorithms", "X448 no deriveKey usage for base key", - "X448 base key is not a private key", "X448 public property value is a private key", "X448 public property value is a secret key" ], "cfrg_curves_keys.https.any.worker.html": [ - "X25519 deriveBits checks for all-zero value result with a key of order 0", - "X25519 deriveBits checks for all-zero value result with a key of order 1", - "X25519 deriveBits checks for all-zero value result with a key of order 8", - "X25519 deriveBits checks for all-zero value result with a key of order p-1 (order 2)", - "X25519 deriveBits checks for all-zero value result with a key of order p (=0, order 4)", - "X25519 deriveBits checks for all-zero value result with a key of order p+1 (=1, order 1)", "X448 deriveBits checks for all-zero value result with a key of order 0", "X448 deriveBits checks for all-zero value result with a key of order 1", "X448 deriveBits checks for all-zero value result with a key of order p-1 (order 2)", "X448 deriveBits checks for all-zero value result with a key of order p (=0, order 4)", "X448 deriveBits checks for all-zero value result with a key of order p+1 (=1, order 1)", - "Key derivation using a X25519 generated keys.", - "Key derivation using a X448 generated keys.", - "X25519 good parameters", - "X25519 mixed case parameters", - "X25519 mismatched algorithms", "X448 good parameters", "X448 mixed case parameters", "X448 mismatched algorithms", "X448 no deriveKey usage for base key", - "X448 base key is not a private key", "X448 public property value is a private key", "X448 public property value is a secret key" ], - "derived_bits_length.https.any.html": [ - "X25519 derivation with 256 as 'length' parameter", - "X25519 derivation with 0 as 'length' parameter", - "X25519 derivation with null as 'length' parameter", - "X25519 derivation with undefined as 'length' parameter", - "X25519 derivation with omitted as 'length' parameter" - ], - "derived_bits_length.https.any.worker.html": [ - "X25519 derivation with 256 as 'length' parameter", - "X25519 derivation with 0 as 'length' parameter", - "X25519 derivation with null as 'length' parameter", - "X25519 derivation with undefined as 'length' parameter", - "X25519 derivation with omitted as 'length' parameter" - ] + "derived_bits_length.https.any.html": true, + "derived_bits_length.https.any.worker.html": true }, "digest": { "digest.https.any.html": true, @@ -819,82 +757,16 @@ ], "failures_X25519.https.any.html": true, "failures_X25519.https.any.worker.html": true, - "failures_X448.https.any.html": [ - "Bad usages: generateKey({name: X448}, true, [encrypt])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, encrypt])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, deriveKey, encrypt])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, encrypt])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits, deriveKey, deriveBits, encrypt])", - "Bad usages: generateKey({name: X448}, true, [decrypt])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, decrypt])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, deriveKey, decrypt])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, decrypt])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits, deriveKey, deriveBits, decrypt])", - "Bad usages: generateKey({name: X448}, true, [sign])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, sign])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, deriveKey, sign])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, sign])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits, deriveKey, deriveBits, sign])", - "Bad usages: generateKey({name: X448}, true, [verify])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, verify])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, deriveKey, verify])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, verify])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits, deriveKey, deriveBits, verify])", - "Bad usages: generateKey({name: X448}, true, [wrapKey])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, wrapKey])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, deriveKey, wrapKey])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, wrapKey])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits, deriveKey, deriveBits, wrapKey])", - "Bad usages: generateKey({name: X448}, true, [unwrapKey])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, unwrapKey])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, deriveKey, unwrapKey])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, unwrapKey])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits, deriveKey, deriveBits, unwrapKey])", - "Empty usages: generateKey({name: X448}, false, [])", - "Empty usages: generateKey({name: X448}, true, [])" - ], - "failures_X448.https.any.worker.html": [ - "Bad usages: generateKey({name: X448}, true, [encrypt])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, encrypt])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, deriveKey, encrypt])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, encrypt])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits, deriveKey, deriveBits, encrypt])", - "Bad usages: generateKey({name: X448}, true, [decrypt])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, decrypt])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, deriveKey, decrypt])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, decrypt])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits, deriveKey, deriveBits, decrypt])", - "Bad usages: generateKey({name: X448}, true, [sign])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, sign])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, deriveKey, sign])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, sign])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits, deriveKey, deriveBits, sign])", - "Bad usages: generateKey({name: X448}, true, [verify])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, verify])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, deriveKey, verify])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, verify])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits, deriveKey, deriveBits, verify])", - "Bad usages: generateKey({name: X448}, true, [wrapKey])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, wrapKey])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, deriveKey, wrapKey])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, wrapKey])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits, deriveKey, deriveBits, wrapKey])", - "Bad usages: generateKey({name: X448}, true, [unwrapKey])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, unwrapKey])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, deriveKey, unwrapKey])", - "Bad usages: generateKey({name: X448}, true, [deriveBits, unwrapKey])", - "Bad usages: generateKey({name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits, deriveKey, deriveBits, unwrapKey])", - "Empty usages: generateKey({name: X448}, false, [])", - "Empty usages: generateKey({name: X448}, true, [])" - ], + "failures_X448.https.any.html": true, + "failures_X448.https.any.worker.html": true, "successes_Ed25519.https.any.html": true, "successes_Ed25519.https.any.worker.html": true, "successes_Ed448.https.any.html": false, "successes_Ed448.https.any.worker.html": false, "successes_X25519.https.any.html": true, "successes_X25519.https.any.worker.html": true, - "successes_X448.https.any.html": false, - "successes_X448.https.any.worker.html": false + "successes_X448.https.any.html": true, + "successes_X448.https.any.worker.html": true }, "historical.any.html": false, "historical.any.worker.html": false, @@ -981,10 +853,6 @@ "Good parameters with ignored JWK alg: X25519 (jwk, object(crv, d, x, kty), {name: X25519}, true, [deriveBits])", "Good parameters: X25519 bits (jwk, object(crv, d, x, kty), {name: X25519}, true, [deriveKey, deriveBits, deriveKey, deriveBits])", "Good parameters with ignored JWK alg: X25519 (jwk, object(crv, d, x, kty), {name: X25519}, true, [deriveKey, deriveBits, deriveKey, deriveBits])", - "Good parameters: X448 bits (spki, buffer(68), {name: X448}, true, [])", - "Good parameters: X448 bits (jwk, object(kty, crv, x), {name: X448}, true, [])", - "Good parameters with ignored JWK alg: X448 (jwk, object(kty, crv, x), {name: X448}, true, [])", - "Good parameters: X448 bits (raw, buffer(56), {name: X448}, true, [])", "Good parameters: X448 bits (pkcs8, buffer(72), {name: X448}, true, [deriveKey])", "Good parameters: X448 bits (jwk, object(crv, d, x, kty), {name: X448}, true, [deriveKey])", "Good parameters with ignored JWK alg: X448 (jwk, object(crv, d, x, kty), {name: X448}, true, [deriveKey])", @@ -997,17 +865,10 @@ "Good parameters: X448 bits (pkcs8, buffer(72), {name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits])", "Good parameters: X448 bits (jwk, object(crv, d, x, kty), {name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits])", "Good parameters with ignored JWK alg: X448 (jwk, object(crv, d, x, kty), {name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits])", - "Good parameters: X448 bits (spki, buffer(68), {name: X448}, false, [])", - "Good parameters: X448 bits (jwk, object(kty, crv, x), {name: X448}, false, [])", - "Good parameters: X448 bits (raw, buffer(56), {name: X448}, false, [])", "Good parameters: X448 bits (pkcs8, buffer(72), {name: X448}, false, [deriveKey])", - "Good parameters: X448 bits (jwk, object(crv, d, x, kty), {name: X448}, false, [deriveKey])", "Good parameters: X448 bits (pkcs8, buffer(72), {name: X448}, false, [deriveBits, deriveKey])", - "Good parameters: X448 bits (jwk, object(crv, d, x, kty), {name: X448}, false, [deriveBits, deriveKey])", "Good parameters: X448 bits (pkcs8, buffer(72), {name: X448}, false, [deriveBits])", - "Good parameters: X448 bits (jwk, object(crv, d, x, kty), {name: X448}, false, [deriveBits])", - "Good parameters: X448 bits (pkcs8, buffer(72), {name: X448}, false, [deriveKey, deriveBits, deriveKey, deriveBits])", - "Good parameters: X448 bits (jwk, object(crv, d, x, kty), {name: X448}, false, [deriveKey, deriveBits, deriveKey, deriveBits])" + "Good parameters: X448 bits (pkcs8, buffer(72), {name: X448}, false, [deriveKey, deriveBits, deriveKey, deriveBits])" ], "okp_importKey.https.any.worker.html": [ "Good parameters: Ed448 bits (spki, buffer(69), {name: Ed448}, true, [verify])", @@ -1049,10 +910,6 @@ "Good parameters with ignored JWK alg: X25519 (jwk, object(crv, d, x, kty), {name: X25519}, true, [deriveBits])", "Good parameters: X25519 bits (jwk, object(crv, d, x, kty), {name: X25519}, true, [deriveKey, deriveBits, deriveKey, deriveBits])", "Good parameters with ignored JWK alg: X25519 (jwk, object(crv, d, x, kty), {name: X25519}, true, [deriveKey, deriveBits, deriveKey, deriveBits])", - "Good parameters: X448 bits (spki, buffer(68), {name: X448}, true, [])", - "Good parameters: X448 bits (jwk, object(kty, crv, x), {name: X448}, true, [])", - "Good parameters with ignored JWK alg: X448 (jwk, object(kty, crv, x), {name: X448}, true, [])", - "Good parameters: X448 bits (raw, buffer(56), {name: X448}, true, [])", "Good parameters: X448 bits (pkcs8, buffer(72), {name: X448}, true, [deriveKey])", "Good parameters: X448 bits (jwk, object(crv, d, x, kty), {name: X448}, true, [deriveKey])", "Good parameters with ignored JWK alg: X448 (jwk, object(crv, d, x, kty), {name: X448}, true, [deriveKey])", @@ -1065,17 +922,10 @@ "Good parameters: X448 bits (pkcs8, buffer(72), {name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits])", "Good parameters: X448 bits (jwk, object(crv, d, x, kty), {name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits])", "Good parameters with ignored JWK alg: X448 (jwk, object(crv, d, x, kty), {name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits])", - "Good parameters: X448 bits (spki, buffer(68), {name: X448}, false, [])", - "Good parameters: X448 bits (jwk, object(kty, crv, x), {name: X448}, false, [])", - "Good parameters: X448 bits (raw, buffer(56), {name: X448}, false, [])", "Good parameters: X448 bits (pkcs8, buffer(72), {name: X448}, false, [deriveKey])", - "Good parameters: X448 bits (jwk, object(crv, d, x, kty), {name: X448}, false, [deriveKey])", "Good parameters: X448 bits (pkcs8, buffer(72), {name: X448}, false, [deriveBits, deriveKey])", - "Good parameters: X448 bits (jwk, object(crv, d, x, kty), {name: X448}, false, [deriveBits, deriveKey])", "Good parameters: X448 bits (pkcs8, buffer(72), {name: X448}, false, [deriveBits])", - "Good parameters: X448 bits (jwk, object(crv, d, x, kty), {name: X448}, false, [deriveBits])", - "Good parameters: X448 bits (pkcs8, buffer(72), {name: X448}, false, [deriveKey, deriveBits, deriveKey, deriveBits])", - "Good parameters: X448 bits (jwk, object(crv, d, x, kty), {name: X448}, false, [deriveKey, deriveBits, deriveKey, deriveBits])" + "Good parameters: X448 bits (pkcs8, buffer(72), {name: X448}, false, [deriveKey, deriveBits, deriveKey, deriveBits])" ], "okp_importKey_failures_Ed25519.https.any.html": [ "Bad key length: importKey(raw, {name: Ed25519}, true, [verify])", @@ -1155,8 +1005,62 @@ "Invalid key pair: importKey(jwk(private), {name: X25519}, true, [deriveBits])", "Invalid key pair: importKey(jwk(private), {name: X25519}, true, [deriveKey, deriveBits, deriveKey, deriveBits])" ], - "okp_importKey_failures_X448.https.any.html": false, - "okp_importKey_failures_X448.https.any.worker.html": false, + "okp_importKey_failures_X448.https.any.html": [ + "Empty usages: importKey(pkcs8, {name: X448}, true, [])", + "Empty usages: importKey(pkcs8, {name: X448}, false, [])", + "Bad key length: importKey(raw, {name: X448}, true, [])", + "Bad key length: importKey(raw, {name: X448}, false, [])", + "Bad key length: importKey(jwk(private), {name: X448}, true, [deriveKey])", + "Bad key length: importKey(jwk(private), {name: X448}, false, [deriveKey])", + "Bad key length: importKey(jwk(private), {name: X448}, true, [deriveBits, deriveKey])", + "Bad key length: importKey(jwk(private), {name: X448}, false, [deriveBits, deriveKey])", + "Bad key length: importKey(jwk(private), {name: X448}, true, [deriveBits])", + "Bad key length: importKey(jwk(private), {name: X448}, false, [deriveBits])", + "Bad key length: importKey(jwk(private), {name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits])", + "Bad key length: importKey(jwk(private), {name: X448}, false, [deriveKey, deriveBits, deriveKey, deriveBits])", + "Bad key length: importKey(jwk (public) , {name: X448}, true, [])", + "Bad key length: importKey(jwk (public) , {name: X448}, false, [])", + "Missing JWK 'x' parameter: importKey(jwk(private), {name: X448}, true, [deriveKey])", + "Missing JWK 'x' parameter: importKey(jwk(private), {name: X448}, false, [deriveKey])", + "Missing JWK 'x' parameter: importKey(jwk(private), {name: X448}, true, [deriveBits, deriveKey])", + "Missing JWK 'x' parameter: importKey(jwk(private), {name: X448}, false, [deriveBits, deriveKey])", + "Missing JWK 'x' parameter: importKey(jwk(private), {name: X448}, true, [deriveBits])", + "Missing JWK 'x' parameter: importKey(jwk(private), {name: X448}, false, [deriveBits])", + "Missing JWK 'x' parameter: importKey(jwk(private), {name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits])", + "Missing JWK 'x' parameter: importKey(jwk(private), {name: X448}, false, [deriveKey, deriveBits, deriveKey, deriveBits])", + "Invalid key pair: importKey(jwk(private), {name: X448}, true, [deriveKey])", + "Invalid key pair: importKey(jwk(private), {name: X448}, true, [deriveBits, deriveKey])", + "Invalid key pair: importKey(jwk(private), {name: X448}, true, [deriveBits])", + "Invalid key pair: importKey(jwk(private), {name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits])" + ], + "okp_importKey_failures_X448.https.any.worker.html": [ + "Empty usages: importKey(pkcs8, {name: X448}, true, [])", + "Empty usages: importKey(pkcs8, {name: X448}, false, [])", + "Bad key length: importKey(raw, {name: X448}, true, [])", + "Bad key length: importKey(raw, {name: X448}, false, [])", + "Bad key length: importKey(jwk(private), {name: X448}, true, [deriveKey])", + "Bad key length: importKey(jwk(private), {name: X448}, false, [deriveKey])", + "Bad key length: importKey(jwk(private), {name: X448}, true, [deriveBits, deriveKey])", + "Bad key length: importKey(jwk(private), {name: X448}, false, [deriveBits, deriveKey])", + "Bad key length: importKey(jwk(private), {name: X448}, true, [deriveBits])", + "Bad key length: importKey(jwk(private), {name: X448}, false, [deriveBits])", + "Bad key length: importKey(jwk(private), {name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits])", + "Bad key length: importKey(jwk(private), {name: X448}, false, [deriveKey, deriveBits, deriveKey, deriveBits])", + "Bad key length: importKey(jwk (public) , {name: X448}, true, [])", + "Bad key length: importKey(jwk (public) , {name: X448}, false, [])", + "Missing JWK 'x' parameter: importKey(jwk(private), {name: X448}, true, [deriveKey])", + "Missing JWK 'x' parameter: importKey(jwk(private), {name: X448}, false, [deriveKey])", + "Missing JWK 'x' parameter: importKey(jwk(private), {name: X448}, true, [deriveBits, deriveKey])", + "Missing JWK 'x' parameter: importKey(jwk(private), {name: X448}, false, [deriveBits, deriveKey])", + "Missing JWK 'x' parameter: importKey(jwk(private), {name: X448}, true, [deriveBits])", + "Missing JWK 'x' parameter: importKey(jwk(private), {name: X448}, false, [deriveBits])", + "Missing JWK 'x' parameter: importKey(jwk(private), {name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits])", + "Missing JWK 'x' parameter: importKey(jwk(private), {name: X448}, false, [deriveKey, deriveBits, deriveKey, deriveBits])", + "Invalid key pair: importKey(jwk(private), {name: X448}, true, [deriveKey])", + "Invalid key pair: importKey(jwk(private), {name: X448}, true, [deriveBits, deriveKey])", + "Invalid key pair: importKey(jwk(private), {name: X448}, true, [deriveBits])", + "Invalid key pair: importKey(jwk(private), {name: X448}, true, [deriveKey, deriveBits, deriveKey, deriveBits])" + ], "crashtests": { "importKey-unsettled-promise.https.any.html": true, "importKey-unsettled-promise.https.any.worker.html": true @@ -1531,18 +1435,28 @@ }, "wrapKey_unwrapKey": { "wrapKey_unwrapKey.https.any.html": [ - "Can wrap and unwrap X25519 private key keys as non-extractable using pkcs8 and AES-CTR", - "Can wrap and unwrap X25519 private key keys as non-extractable using pkcs8 and AES-CBC", - "Can wrap and unwrap X25519 private key keys as non-extractable using pkcs8 and AES-GCM", - "Can wrap and unwrap X25519 private key keys as non-extractable using pkcs8 and AES-KW", - "Can wrap and unwrap X25519 private key keys as non-extractable using pkcs8 and RSA-OAEP" + "Can wrap and unwrap X448 private key keys using pkcs8 and AES-CTR", + "Can wrap and unwrap X448 private key keys as non-extractable using pkcs8 and AES-CTR", + "Can wrap and unwrap X448 private key keys using pkcs8 and AES-CBC", + "Can wrap and unwrap X448 private key keys as non-extractable using pkcs8 and AES-CBC", + "Can wrap and unwrap X448 private key keys using pkcs8 and AES-GCM", + "Can wrap and unwrap X448 private key keys as non-extractable using pkcs8 and AES-GCM", + "Can wrap and unwrap X448 private key keys using pkcs8 and AES-KW", + "Can wrap and unwrap X448 private key keys as non-extractable using pkcs8 and AES-KW", + "Can wrap and unwrap X448 private key keys using pkcs8 and RSA-OAEP", + "Can wrap and unwrap X448 private key keys as non-extractable using pkcs8 and RSA-OAEP" ], "wrapKey_unwrapKey.https.any.worker.html": [ - "Can wrap and unwrap X25519 private key keys as non-extractable using pkcs8 and AES-CTR", - "Can wrap and unwrap X25519 private key keys as non-extractable using pkcs8 and AES-CBC", - "Can wrap and unwrap X25519 private key keys as non-extractable using pkcs8 and AES-GCM", - "Can wrap and unwrap X25519 private key keys as non-extractable using pkcs8 and AES-KW", - "Can wrap and unwrap X25519 private key keys as non-extractable using pkcs8 and RSA-OAEP" + "Can wrap and unwrap X448 private key keys using pkcs8 and AES-CTR", + "Can wrap and unwrap X448 private key keys as non-extractable using pkcs8 and AES-CTR", + "Can wrap and unwrap X448 private key keys using pkcs8 and AES-CBC", + "Can wrap and unwrap X448 private key keys as non-extractable using pkcs8 and AES-CBC", + "Can wrap and unwrap X448 private key keys using pkcs8 and AES-GCM", + "Can wrap and unwrap X448 private key keys as non-extractable using pkcs8 and AES-GCM", + "Can wrap and unwrap X448 private key keys using pkcs8 and AES-KW", + "Can wrap and unwrap X448 private key keys as non-extractable using pkcs8 and AES-KW", + "Can wrap and unwrap X448 private key keys using pkcs8 and RSA-OAEP", + "Can wrap and unwrap X448 private key keys as non-extractable using pkcs8 and RSA-OAEP" ] } }, @@ -3343,8 +3257,14 @@ "owning-type-message-port.any.worker.html": false, "owning-type.any.html": false, "owning-type.any.worker.html": false, - "from.any.html": true, - "from.any.worker.html": true + "from.any.html": [ + "ReadableStream.from ignores a null @@asyncIterator", + "ReadableStream.from accepts a string" + ], + "from.any.worker.html": [ + "ReadableStream.from ignores a null @@asyncIterator", + "ReadableStream.from accepts a string" + ] }, "transform-streams": { "backpressure.any.html": true, @@ -3821,18 +3741,12 @@ "ab--c.ß (using .hostname)", "ab--c.ß (using .host)", "ab--c.ß (using .hostname)", - "‍.example (using URL)", - "‍.example (using URL.host)", - "‍.example (using URL.hostname)", "‍.example (using )", "‍.example (using .host)", "‍.example (using .hostname)", "‍.example (using )", "‍.example (using .host)", "‍.example (using .hostname)", - "xn--1ug.example (using URL)", - "xn--1ug.example (using URL.host)", - "xn--1ug.example (using URL.hostname)", "xn--1ug.example (using )", "xn--1ug.example (using .host)", "xn--1ug.example (using .hostname)", @@ -3954,18 +3868,8 @@ ], "url-searchparams.any.html": true, "url-searchparams.any.worker.html": true, - "url-setters-stripping.any.html": [ - "Setting port with leading U+0000 (https:)", - "Setting port with leading U+001F (https:)", - "Setting port with leading U+0000 (wpt++:)", - "Setting port with leading U+001F (wpt++:)" - ], - "url-setters-stripping.any.worker.html": [ - "Setting port with leading U+0000 (https:)", - "Setting port with leading U+001F (https:)", - "Setting port with leading U+0000 (wpt++:)", - "Setting port with leading U+001F (wpt++:)" - ], + "url-setters-stripping.any.html": true, + "url-setters-stripping.any.worker.html": true, "url-tojson.any.html": true, "url-tojson.any.worker.html": true, "urlencoded-parser.any.html": true, @@ -3974,12 +3878,8 @@ "urlsearchparams-append.any.worker.html": true, "urlsearchparams-constructor.any.html": true, "urlsearchparams-constructor.any.worker.html": true, - "urlsearchparams-delete.any.html": [ - "Changing the query of a URL with an opaque path can impact the path if the URL has no fragment" - ], - "urlsearchparams-delete.any.worker.html": [ - "Changing the query of a URL with an opaque path can impact the path if the URL has no fragment" - ], + "urlsearchparams-delete.any.html": true, + "urlsearchparams-delete.any.worker.html": true, "urlsearchparams-foreach.any.html": true, "urlsearchparams-foreach.any.worker.html": true, "urlsearchparams-get.any.html": true, @@ -4012,182 +3912,7 @@ "Input − with encoding utf-8", "Input á| with encoding utf-8" ], - "IdnaTestV2.window.html": [ - "ToASCII(\"a‌b\") C1", - "ToASCII(\"A‌B\") C1", - "ToASCII(\"A‌b\") C1", - "ToASCII(\"xn--ab-j1t\") C1", - "ToASCII(\"a‍b\") C2", - "ToASCII(\"A‍B\") C2", - "ToASCII(\"A‍b\") C2", - "ToASCII(\"xn--ab-m1t\") C2", - "ToASCII(\"1.aß‌‍b‌‍cßßßßdςσßßßßßßßßeßßßßßßßßßßxßßßßßßßßßßyßßßßßßßß̂ßz\") C1; C2; A4_2 (ignored)", - "ToASCII(\"1.ASS‌‍B‌‍CSSSSSSSSDΣΣSSSSSSSSSSSSSSSSESSSSSSSSSSSSSSSSSSSSXSSSSSSSSSSSSSSSSSSSSYSSSSSSSSSSSSSSSŜSSZ\") C1; C2; A4_2 (ignored)", - "ToASCII(\"1.ASS‌‍B‌‍CSSSSSSSSDΣΣSSSSSSSSSSSSSSSSESSSSSSSSSSSSSSSSSSSSXSSSSSSSSSSSSSSSSSSSSYSSSSSSSSSSSSSSSŜSSZ\") C1; C2; A4_2 (ignored)", - "ToASCII(\"1.ass‌‍b‌‍cssssssssdσσssssssssssssssssessssssssssssssssssssxssssssssssssssssssssysssssssssssssssŝssz\") C1; C2; A4_2 (ignored)", - "ToASCII(\"1.ass‌‍b‌‍cssssssssdσσssssssssssssssssessssssssssssssssssssxssssssssssssssssssssysssssssssssssssŝssz\") C1; C2; A4_2 (ignored)", - "ToASCII(\"1.Ass‌‍b‌‍cssssssssdσσssssssssssssssssessssssssssssssssssssxssssssssssssssssssssysssssssssssssssŝssz\") C1; C2; A4_2 (ignored)", - "ToASCII(\"1.Ass‌‍b‌‍cssssssssdσσssssssssssssssssessssssssssssssssssssxssssssssssssssssssssysssssssssssssssŝssz\") C1; C2; A4_2 (ignored)", - "ToASCII(\"1.xn--assbcssssssssdssssssssssssssssessssssssssssssssssssxssssssssssssssssssssysssssssssssssssssz-pxq1419aa69989dba9gc\") C1; C2; A4_2 (ignored)", - "ToASCII(\"1.Aß‌‍b‌‍cßßßßdςσßßßßßßßßeßßßßßßßßßßxßßßßßßßßßßyßßßßßßßß̂ßz\") C1; C2; A4_2 (ignored)", - "ToASCII(\"1.xn--abcdexyz-qyacaaabaaaaaaabaaaaaaaaabaaaaaaaaabaaaaaaaa010ze2isb1140zba8cc\") C1; C2; A4_2 (ignored)", - "ToASCII(\"‌x‍n‌-‍-bß\") C1; C2", - "ToASCII(\"‌X‍N‌-‍-BSS\") C1; C2", - "ToASCII(\"‌x‍n‌-‍-bss\") C1; C2", - "ToASCII(\"‌X‍n‌-‍-Bss\") C1; C2", - "ToASCII(\"xn--xn--bss-7z6ccid\") C1; C2", - "ToASCII(\"‌X‍n‌-‍-Bß\") C1; C2", - "ToASCII(\"xn--xn--b-pqa5796ccahd\") C1; C2", - "ToASCII(\"ஹ‍\") C2", - "ToASCII(\"xn--dmc225h\") C2", - "ToASCII(\"‍\") C2", - "ToASCII(\"xn--1ug\") C2", - "ToASCII(\"ஹ‌\") C1", - "ToASCII(\"xn--dmc025h\") C1", - "ToASCII(\"‌\") C1", - "ToASCII(\"xn--0ug\") C1", - "ToASCII(\"ۯ‌ۯ\") C1", - "ToASCII(\"xn--cmba004q\") C1", - "ToASCII(\"ß۫。‍\") C2", - "ToASCII(\"SS۫。‍\") C2", - "ToASCII(\"ss۫。‍\") C2", - "ToASCII(\"Ss۫。‍\") C2", - "ToASCII(\"xn--ss-59d.xn--1ug\") C2", - "ToASCII(\"xn--zca012a.xn--1ug\") C2", - "ToASCII(\"‌긃.榶-\") C1; V3 (ignored)", - "ToASCII(\"‌긃.榶-\") C1; V3 (ignored)", - "ToASCII(\"xn--0ug3307c.xn----d87b\") C1; V3 (ignored)", - "ToASCII(\"Å둄-.‌\") C1; V3 (ignored)", - "ToASCII(\"Å둄-.‌\") C1; V3 (ignored)", - "ToASCII(\"Å둄-.‌\") C1; V3 (ignored)", - "ToASCII(\"Å둄-.‌\") C1; V3 (ignored)", - "ToASCII(\"å둄-.‌\") C1; V3 (ignored)", - "ToASCII(\"å둄-.‌\") C1; V3 (ignored)", - "ToASCII(\"xn----1fa1788k.xn--0ug\") C1; V3 (ignored)", - "ToASCII(\"å둄-.‌\") C1; V3 (ignored)", - "ToASCII(\"å둄-.‌\") C1; V3 (ignored)", - "ToASCII(\"ꡦᡑ‍1.。𐋣-\") C2; V3 (ignored); A4_2 (ignored)", - "ToASCII(\"xn--1-o7j663bdl7m..xn----381i\") C2; V3 (ignored); A4_2 (ignored)", - "ToASCII(\"1.䰹‍-。웈\") C2; V3 (ignored)", - "ToASCII(\"1.䰹‍-。웈\") C2; V3 (ignored)", - "ToASCII(\"1.xn----tgnz80r.xn--kp5b\") C2; V3 (ignored)", - "ToASCII(\"-3.‍ヌᢕ\") C2; V3 (ignored)", - "ToASCII(\"-3.xn--fbf739aq5o\") C2; V3 (ignored)", - "ToASCII(\"ς-。‌𝟭-\") C1; V3 (ignored)", - "ToASCII(\"ς-。‌1-\") C1; V3 (ignored)", - "ToASCII(\"Σ-。‌1-\") C1; V3 (ignored)", - "ToASCII(\"σ-。‌1-\") C1; V3 (ignored)", - "ToASCII(\"xn----zmb.xn--1--i1t\") C1; V3 (ignored)", - "ToASCII(\"xn----xmb.xn--1--i1t\") C1; V3 (ignored)", - "ToASCII(\"Σ-。‌𝟭-\") C1; V3 (ignored)", - "ToASCII(\"σ-。‌𝟭-\") C1; V3 (ignored)", - "ToASCII(\"ᡯ⚉姶🄉.۷‍🎪‍\") C2; P1; V6", - "ToASCII(\"𝟵隁⯮.᠍‌\") C1", - "ToASCII(\"9隁⯮.᠍‌\") C1", - "ToASCII(\"xn--9-mfs8024b.xn--0ug\") C1", - "ToASCII(\"ß‌꫶ᢥ.⊶ⴡⴖ\") C1", - "ToASCII(\"ss‌꫶ᢥ.⊶ⴡⴖ\") C1", - "ToASCII(\"xn--ss-4ep585bkm5p.xn--ifh802b6a\") C1", - "ToASCII(\"xn--zca682johfi89m.xn--ifh802b6a\") C1", - "ToASCII(\"ß‌꫶ᢥ.⊶ⴡⴖ\") C1", - "ToASCII(\"ss‌꫶ᢥ.⊶ⴡⴖ\") C1", - "ToASCII(\"-。‍\") C2; V3 (ignored)", - "ToASCII(\"-。‍\") C2; V3 (ignored)", - "ToASCII(\"-.xn--1ug\") C2; V3 (ignored)", - "ToASCII(\"ς‍-.ⴣ𦟙\") C2; V3 (ignored)", - "ToASCII(\"σ‍-.ⴣ𦟙\") C2; V3 (ignored)", - "ToASCII(\"xn----zmb048s.xn--rlj2573p\") C2; V3 (ignored)", - "ToASCII(\"xn----xmb348s.xn--rlj2573p\") C2; V3 (ignored)", - "ToASCII(\"鱊。‌\") C1", - "ToASCII(\"xn--rt6a.xn--0ug\") C1", - "ToASCII(\"‌ⴚ。ς\") C1", - "ToASCII(\"‌ⴚ。σ\") C1", - "ToASCII(\"xn--0ug262c.xn--4xa\") C1", - "ToASCII(\"xn--0ug262c.xn--3xa\") C1", - "ToASCII(\"‌ⴚ。ς\") C1", - "ToASCII(\"‌ⴚ。σ\") C1", - "ToASCII(\"‍⾕。‌꥓̐ꡎ\") C1; C2", - "ToASCII(\"‍⾕。‌꥓̐ꡎ\") C1; C2", - "ToASCII(\"‍谷。‌꥓̐ꡎ\") C1; C2", - "ToASCII(\"xn--1ug0273b.xn--0sa359l6n7g13a\") C1; C2", - "ToASCII(\"‍。‌\") C1; C2", - "ToASCII(\"xn--1ug.xn--0ug\") C1; C2", - "ToASCII(\"‌。。\") C1; A4_2 (ignored)", - "ToASCII(\"xn--0ug..\") C1; A4_2 (ignored)", - "ToASCII(\"ᡲ-𝟹.ß-‌-\") C1; V3 (ignored)", - "ToASCII(\"ᡲ-3.ß-‌-\") C1; V3 (ignored)", - "ToASCII(\"ᡲ-3.SS-‌-\") C1; V3 (ignored)", - "ToASCII(\"ᡲ-3.ss-‌-\") C1; V3 (ignored)", - "ToASCII(\"ᡲ-3.Ss-‌-\") C1; V3 (ignored)", - "ToASCII(\"xn---3-p9o.xn--ss---276a\") C1; V3 (ignored)", - "ToASCII(\"xn---3-p9o.xn-----fia9303a\") C1; V3 (ignored)", - "ToASCII(\"ᡲ-𝟹.SS-‌-\") C1; V3 (ignored)", - "ToASCII(\"ᡲ-𝟹.ss-‌-\") C1; V3 (ignored)", - "ToASCII(\"ᡲ-𝟹.Ss-‌-\") C1; V3 (ignored)", - "ToASCII(\"𝟙。‍𝟸‍⁷\") C2", - "ToASCII(\"1。‍2‍7\") C2", - "ToASCII(\"1.xn--27-l1tb\") C2", - "ToASCII(\"‌.ßⴉ-\") C1; V3 (ignored)", - "ToASCII(\"‌.ssⴉ-\") C1; V3 (ignored)", - "ToASCII(\"‌.Ssⴉ-\") C1; V3 (ignored)", - "ToASCII(\"xn--0ug.xn--ss--bi1b\") C1; V3 (ignored)", - "ToASCII(\"xn--0ug.xn----pfa2305a\") C1; V3 (ignored)", - "ToASCII(\"ⴏ󠅋-.‍ⴉ\") C2; V3 (ignored)", - "ToASCII(\"xn----3vs.xn--1ug532c\") C2; V3 (ignored)", - "ToASCII(\"ⴏ󠅋-.‍ⴉ\") C2; V3 (ignored)", - "ToASCII(\"。ⴖͦ.‌\") C1; A4_2 (ignored)", - "ToASCII(\".xn--hva754s.xn--0ug\") C1; A4_2 (ignored)", - "ToASCII(\"‍攌꯭。ᢖ-ⴘ\") C2", - "ToASCII(\"xn--1ug592ykp6b.xn----mck373i\") C2", - "ToASCII(\"‌ꖨ.16.3툒۳\") C1", - "ToASCII(\"‌ꖨ.16.3툒۳\") C1", - "ToASCII(\"xn--0ug2473c.16.xn--3-nyc0117m\") C1", - "ToASCII(\"𝟏𝨙⸖.‍\") C2", - "ToASCII(\"1𝨙⸖.‍\") C2", - "ToASCII(\"xn--1-5bt6845n.xn--1ug\") C2", - "ToASCII(\"-‍.ⴞ𐋷\") C2; V3 (ignored)", - "ToASCII(\"xn----ugn.xn--mlj8559d\") C2; V3 (ignored)", - "ToASCII(\"嬃𝍌.‍ୄ\") C2", - "ToASCII(\"嬃𝍌.‍ୄ\") C2", - "ToASCII(\"xn--b6s0078f.xn--0ic557h\") C2", - "ToASCII(\"‍.F\") C2", - "ToASCII(\"‍.f\") C2", - "ToASCII(\"xn--1ug.f\") C2", - "ToASCII(\"‍㨲。ß\") C2", - "ToASCII(\"‍㨲。ß\") C2", - "ToASCII(\"‍㨲。SS\") C2", - "ToASCII(\"‍㨲。ss\") C2", - "ToASCII(\"‍㨲。Ss\") C2", - "ToASCII(\"xn--1ug914h.ss\") C2", - "ToASCII(\"xn--1ug914h.xn--zca\") C2", - "ToASCII(\"‍㨲。SS\") C2", - "ToASCII(\"‍㨲。ss\") C2", - "ToASCII(\"‍㨲。Ss\") C2", - "ToASCII(\"璼𝨭。‌󠇟\") C1", - "ToASCII(\"璼𝨭。‌󠇟\") C1", - "ToASCII(\"xn--gky8837e.xn--0ug\") C1", - "ToASCII(\"‌.‌\") C1", - "ToASCII(\"xn--0ug.xn--0ug\") C1", - "ToASCII(\"𝟠4󠇗𝈻.‍𐋵⛧‍\") C2", - "ToASCII(\"84󠇗𝈻.‍𐋵⛧‍\") C2", - "ToASCII(\"xn--84-s850a.xn--1uga573cfq1w\") C2", - "ToASCII(\"‍‌󠆪。ß𑓃\") C1; C2", - "ToASCII(\"‍‌󠆪。ß𑓃\") C1; C2", - "ToASCII(\"‍‌󠆪。SS𑓃\") C1; C2", - "ToASCII(\"‍‌󠆪。ss𑓃\") C1; C2", - "ToASCII(\"‍‌󠆪。Ss𑓃\") C1; C2", - "ToASCII(\"xn--0ugb.xn--ss-bh7o\") C1; C2", - "ToASCII(\"xn--0ugb.xn--zca0732l\") C1; C2", - "ToASCII(\"‍‌󠆪。SS𑓃\") C1; C2", - "ToASCII(\"‍‌󠆪。ss𑓃\") C1; C2", - "ToASCII(\"‍‌󠆪。Ss𑓃\") C1; C2", - "ToASCII(\"。‌ヶ䒩.ꡪ\") C1; A4_2 (ignored)", - "ToASCII(\".xn--0ug287dj0o.xn--gd9a\") C1; A4_2 (ignored)", - "ToASCII(\"梉。‌\") C1", - "ToASCII(\"xn--7zv.xn--0ug\") C1", - "ToASCII(\"𐋷。‍\") C2", - "ToASCII(\"xn--r97c.xn--1ug\") C2" - ], + "IdnaTestV2.window.html": true, "javascript-urls.window.html": false, "url-constructor.any.html?exclude=(file|javascript|mailto)": [ "Parsing: without base", @@ -4840,21 +4565,11 @@ ": Setting .pathname = '/foo' Opaque paths cannot be set" ], "url-setters.any.html?exclude=(file|javascript|mailto)": [ - "URL: Setting .host = 'example.com:invalid' Anything other than ASCII digit stops the port parser in a setter but is not an error", - "URL: Setting .host = '[::1]:invalid' Anything other than ASCII digit stops the port parser in a setter but is not an error", "URL: Setting .hostname = 'example.com:8080' : delimiter invalidates entire value", "URL: Setting .hostname = 'example.com:' : delimiter invalidates entire value", "URL: Setting .hostname = 'h' Drop /. from path", "URL: Setting .hostname = ''", - "URL: Setting .port = 'randomstring' Setting port to a string that doesn't parse as a number", - "URL: Setting .pathname = '' Non-special URLs can have their paths erased", - "URL: Setting .pathname = '' Non-special URLs with an empty host can have their paths erased", - "URL: Setting .pathname = '/.//p' Serialize /. in path", - "URL: Setting .pathname = '/..//p'", - "URL: Setting .pathname = '//p'", - "URL: Setting .pathname = 'p' Drop /. from path", - "URL: Setting .search = '' Do not drop trailing spaces from non-trailing opaque paths", - "URL: Setting .search = ''" + "URL: Setting .pathname = '' Non-special URLs with an empty host can have their paths erased" ], "url-setters.any.html?include=file": [ "URL: Setting .pathname = '\\\\' File URLs and (back)slashes", @@ -4864,21 +4579,11 @@ "url-setters.any.html?include=javascript": true, "url-setters.any.html?include=mailto": true, "url-setters.any.worker.html?exclude=(file|javascript|mailto)": [ - "URL: Setting .host = 'example.com:invalid' Anything other than ASCII digit stops the port parser in a setter but is not an error", - "URL: Setting .host = '[::1]:invalid' Anything other than ASCII digit stops the port parser in a setter but is not an error", "URL: Setting .hostname = 'example.com:8080' : delimiter invalidates entire value", "URL: Setting .hostname = 'example.com:' : delimiter invalidates entire value", "URL: Setting .hostname = 'h' Drop /. from path", "URL: Setting .hostname = ''", - "URL: Setting .port = 'randomstring' Setting port to a string that doesn't parse as a number", - "URL: Setting .pathname = '' Non-special URLs can have their paths erased", - "URL: Setting .pathname = '' Non-special URLs with an empty host can have their paths erased", - "URL: Setting .pathname = '/.//p' Serialize /. in path", - "URL: Setting .pathname = '/..//p'", - "URL: Setting .pathname = '//p'", - "URL: Setting .pathname = 'p' Drop /. from path", - "URL: Setting .search = '' Do not drop trailing spaces from non-trailing opaque paths", - "URL: Setting .search = ''" + "URL: Setting .pathname = '' Non-special URLs with an empty host can have their paths erased" ], "url-setters.any.worker.html?include=file": [ "URL: Setting .pathname = '\\\\' File URLs and (back)slashes", @@ -10084,7 +9789,7 @@ "event_constructor.window.html": false, "event_initstorageevent.window.html": false, "missing_arguments.window.html": true, - "storage_builtins.window.html": true, + "storage_builtins.window.html": false, "storage_clear.window.html": true, "storage_functions_not_overwritten.window.html": true, "storage_getitem.window.html": true, @@ -11598,10 +11303,10 @@ "006.html?default": true, "006.html?wpt_flags=h2": false, "006.html?wss": false, - "007.html?default": false, + "007.html?default": true, "007.html?wpt_flags=h2": false, "007.html?wss": false, - "008.html?default": false, + "008.html?default": true, "008.html?wss": false, "009.html?default": { "ignore": true @@ -11711,9 +11416,6 @@ "ctor-1.html": false, "ctor-null.html": false, "ctor-undefined.html": false, - "same-origin.html": [ - "non-parsable URL" - ], "terminate.html": true, "use-base-url.html": false }, @@ -12297,18 +11999,8 @@ "Component: hash Left: {\"hash\":\"a\"} Right: {\"hash\":\"b\"}" ], "urlpattern.any.html": [ - "Pattern: [{\"pathname\":\"/foo/:bar?\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/:bar*\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/(.*)?\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/*?\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/(.*)*\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/**\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [\"https://(sub.)?example.com/foo\"] Inputs: [\"https://example.com/foo\"]", - "Pattern: [\"https://(sub(?:.))?example.com/foo\"] Inputs: [\"https://example.com/foo\"]", - "Pattern: [\"data\\\\:text/javascript,let x = 100/:tens?5;\"] Inputs: [\"data:text/javascript,let x = 100/5;\"]", "Pattern: [{\"hostname\":\"bad\\\\:hostname\"}] Inputs: undefined", "Pattern: [] Inputs: []", - "Pattern: [{\"pathname\":\"*{}**?\"}] Inputs: [{\"pathname\":\"foobar\"}]", "Pattern: [{\"pathname\":\"/foo/bar\"},{\"ignoreCase\":true}] Inputs: [{\"pathname\":\"/FOO/BAR\"}]", "Pattern: [\"https://example.com:8080/foo?bar#baz\",{\"ignoreCase\":true}] Inputs: [{\"pathname\":\"/FOO\",\"search\":\"BAR\",\"hash\":\"BAZ\",\"baseURL\":\"https://example.com:8080\"}]", "Pattern: [{\"pathname\":\"/([[a-z]--a])\"}] Inputs: [{\"pathname\":\"/a\"}]", @@ -12317,18 +12009,8 @@ "Pattern: [{\"pathname\":\"/([\\\\d&&[0-1]])\"}] Inputs: [{\"pathname\":\"/3\"}]" ], "urlpattern.any.worker.html": [ - "Pattern: [{\"pathname\":\"/foo/:bar?\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/:bar*\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/(.*)?\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/*?\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/(.*)*\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/**\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [\"https://(sub.)?example.com/foo\"] Inputs: [\"https://example.com/foo\"]", - "Pattern: [\"https://(sub(?:.))?example.com/foo\"] Inputs: [\"https://example.com/foo\"]", - "Pattern: [\"data\\\\:text/javascript,let x = 100/:tens?5;\"] Inputs: [\"data:text/javascript,let x = 100/5;\"]", "Pattern: [{\"hostname\":\"bad\\\\:hostname\"}] Inputs: undefined", "Pattern: [] Inputs: []", - "Pattern: [{\"pathname\":\"*{}**?\"}] Inputs: [{\"pathname\":\"foobar\"}]", "Pattern: [{\"pathname\":\"/foo/bar\"},{\"ignoreCase\":true}] Inputs: [{\"pathname\":\"/FOO/BAR\"}]", "Pattern: [\"https://example.com:8080/foo?bar#baz\",{\"ignoreCase\":true}] Inputs: [{\"pathname\":\"/FOO\",\"search\":\"BAR\",\"hash\":\"BAZ\",\"baseURL\":\"https://example.com:8080\"}]", "Pattern: [{\"pathname\":\"/([[a-z]--a])\"}] Inputs: [{\"pathname\":\"/a\"}]", @@ -12337,18 +12019,8 @@ "Pattern: [{\"pathname\":\"/([\\\\d&&[0-1]])\"}] Inputs: [{\"pathname\":\"/3\"}]" ], "urlpattern.https.any.html": [ - "Pattern: [{\"pathname\":\"/foo/:bar?\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/:bar*\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/(.*)?\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/*?\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/(.*)*\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/**\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [\"https://(sub.)?example.com/foo\"] Inputs: [\"https://example.com/foo\"]", - "Pattern: [\"https://(sub(?:.))?example.com/foo\"] Inputs: [\"https://example.com/foo\"]", - "Pattern: [\"data\\\\:text/javascript,let x = 100/:tens?5;\"] Inputs: [\"data:text/javascript,let x = 100/5;\"]", "Pattern: [{\"hostname\":\"bad\\\\:hostname\"}] Inputs: undefined", "Pattern: [] Inputs: []", - "Pattern: [{\"pathname\":\"*{}**?\"}] Inputs: [{\"pathname\":\"foobar\"}]", "Pattern: [{\"pathname\":\"/foo/bar\"},{\"ignoreCase\":true}] Inputs: [{\"pathname\":\"/FOO/BAR\"}]", "Pattern: [\"https://example.com:8080/foo?bar#baz\",{\"ignoreCase\":true}] Inputs: [{\"pathname\":\"/FOO\",\"search\":\"BAR\",\"hash\":\"BAZ\",\"baseURL\":\"https://example.com:8080\"}]", "Pattern: [{\"pathname\":\"/([[a-z]--a])\"}] Inputs: [{\"pathname\":\"/a\"}]", @@ -12357,18 +12029,8 @@ "Pattern: [{\"pathname\":\"/([\\\\d&&[0-1]])\"}] Inputs: [{\"pathname\":\"/3\"}]" ], "urlpattern.https.any.worker.html": [ - "Pattern: [{\"pathname\":\"/foo/:bar?\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/:bar*\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/(.*)?\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/*?\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/(.*)*\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [{\"pathname\":\"/foo/**\"}] Inputs: [{\"pathname\":\"/foo\"}]", - "Pattern: [\"https://(sub.)?example.com/foo\"] Inputs: [\"https://example.com/foo\"]", - "Pattern: [\"https://(sub(?:.))?example.com/foo\"] Inputs: [\"https://example.com/foo\"]", - "Pattern: [\"data\\\\:text/javascript,let x = 100/:tens?5;\"] Inputs: [\"data:text/javascript,let x = 100/5;\"]", "Pattern: [{\"hostname\":\"bad\\\\:hostname\"}] Inputs: undefined", "Pattern: [] Inputs: []", - "Pattern: [{\"pathname\":\"*{}**?\"}] Inputs: [{\"pathname\":\"foobar\"}]", "Pattern: [{\"pathname\":\"/foo/bar\"},{\"ignoreCase\":true}] Inputs: [{\"pathname\":\"/FOO/BAR\"}]", "Pattern: [\"https://example.com:8080/foo?bar#baz\",{\"ignoreCase\":true}] Inputs: [{\"pathname\":\"/FOO\",\"search\":\"BAR\",\"hash\":\"BAZ\",\"baseURL\":\"https://example.com:8080\"}]", "Pattern: [{\"pathname\":\"/([[a-z]--a])\"}] Inputs: [{\"pathname\":\"/a\"}]", diff --git a/tests/wpt/runner/utils.ts b/tests/wpt/runner/utils.ts index 1674419cd1833f..140c388ec2118b 100644 --- a/tests/wpt/runner/utils.ts +++ b/tests/wpt/runner/utils.ts @@ -76,7 +76,10 @@ export function getManifest(): Manifest { /// WPT TEST EXPECTATIONS -const EXPECTATION_PATH = join(ROOT_PATH, "./tests/wpt/runner/expectation.json"); +export const EXPECTATION_PATH = join( + ROOT_PATH, + "./tests/wpt/runner/expectation.json", +); export interface Expectation { [key: string]: Expectation | boolean | string[] | { ignore: boolean }; @@ -87,9 +90,12 @@ export function getExpectation(): Expectation { return JSON.parse(expectationText); } -export function saveExpectation(expectation: Expectation) { +export function saveExpectation( + expectation: Expectation, + path: string = EXPECTATION_PATH, +) { Deno.writeTextFileSync( - EXPECTATION_PATH, + path, JSON.stringify(expectation, undefined, " ") + "\n", ); } @@ -134,6 +140,15 @@ export function runPy>( }).spawn(); } +export async function runGitDiff(args: string[]): string { + await new Deno.Command("git", { + args: ["diff", ...args], + stdout: "inherit", + stderr: "inherit", + cwd: ROOT_PATH, + }).output(); +} + export async function checkPy3Available() { const { success, stdout } = await runPy(["--version"], { stdout: "piped", diff --git a/tests/wpt/wpt.ts b/tests/wpt/wpt.ts index c42ff51e6eba79..b13a10cf4c3690 100755 --- a/tests/wpt/wpt.ts +++ b/tests/wpt/wpt.ts @@ -18,6 +18,7 @@ import { checkPy3Available, escapeLoneSurrogates, Expectation, + EXPECTATION_PATH, generateRunInfo, getExpectation, getExpectFailForCase, @@ -30,6 +31,7 @@ import { noIgnore, quiet, rest, + runGitDiff, runPy, updateManifest, wptreport, @@ -256,7 +258,16 @@ async function run() { await Deno.writeTextFile(wptreport, JSON.stringify(report) + "\n"); } + const newExpectations = newExpectation(results); + const tmp = Deno.makeTempFileSync(); + saveExpectation(newExpectations, tmp); + const code = reportFinal(results, endTime - startTime); + + // Run git diff to see what changed + await runGitDiff([EXPECTATION_PATH, tmp]); + Deno.removeSync(tmp); + Deno.exit(code); } @@ -390,6 +401,19 @@ async function update() { await Deno.writeTextFile(json, JSON.stringify(results) + "\n"); } + const newExpectations = newExpectation(results); + saveExpectation(newExpectations); + + reportFinal(results, endTime - startTime); + + console.log(blue("Updated expectation.json to match reality.")); + + Deno.exit(0); +} + +function newExpectation( + results: { test: TestToRun; result: TestResult }[], +): Expectation { const resultTests: Record< string, { passed: string[]; failed: string[]; testSucceeded: boolean } @@ -431,13 +455,7 @@ async function update() { ); } - saveExpectation(currentExpectation); - - reportFinal(results, endTime - startTime); - - console.log(blue("Updated expectation.json to match reality.")); - - Deno.exit(0); + return currentExpectation; } function insertExpectation( diff --git a/tools/copyright_checker.js b/tools/copyright_checker.js index 24afe1dfd73eec..d7d196bc44656b 100755 --- a/tools/copyright_checker.js +++ b/tools/copyright_checker.js @@ -31,6 +31,8 @@ export async function checkCopyright() { ":!:.github/mtime_cache/action.js", ":!:cli/bench/testdata/**", ":!:cli/tools/bench/mitata.rs", + ":!:cli/tools/doc/prism.css", + ":!:cli/tools/doc/prism.js", ":!:cli/tools/init/templates/**", ":!:cli/tsc/*typescript.js", ":!:cli/tsc/compiler.d.ts", diff --git a/tools/core_import_map.json b/tools/core_import_map.json index aae4e63a4593d4..bc0674277e8017 100644 --- a/tools/core_import_map.json +++ b/tools/core_import_map.json @@ -248,6 +248,8 @@ "ext:runtime/90_deno_ns.js": "../runtime/js/90_deno_ns.js", "ext:runtime/98_global_scope.js": "../runtime/js/98_global_scope.js", "ext:deno_node/_util/std_fmt_colors.ts": "../ext/node/polyfills/_util/std_fmt_colors.ts", + "ext:deno_telemetry/telemetry.ts": "../ext/deno_telemetry/telemetry.ts", + "ext:deno_telemetry/util.ts": "../ext/deno_telemetry/util.ts", "@std/archive": "../tests/util/std/archive/mod.ts", "@std/archive/tar": "../tests/util/std/archive/tar.ts", "@std/archive/untar": "../tests/util/std/archive/untar.ts", diff --git a/tools/format.js b/tools/format.js index 74c608644bf968..b29667ca77a6a2 100755 --- a/tools/format.js +++ b/tools/format.js @@ -1,4 +1,4 @@ -#!/usr/bin/env -S deno run --allow-write --allow-read --allow-run=deno --allow-net --config=tests/config/deno.json +#!/usr/bin/env -S deno run --allow-all --config=tests/config/deno.json // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. import { join, ROOT_PATH } from "./util.js"; diff --git a/tools/lint.js b/tools/lint.js index 6cb27f8dc9c1de..2312cde272580e 100755 --- a/tools/lint.js +++ b/tools/lint.js @@ -1,9 +1,18 @@ -#!/usr/bin/env -S deno run --allow-write --allow-read --allow-run --allow-net --config=tests/config/deno.json +#!/usr/bin/env -S deno run --allow-all --config=tests/config/deno.json // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. // deno-lint-ignore-file no-console -import { buildMode, getPrebuilt, getSources, join, ROOT_PATH } from "./util.js"; +import { + buildMode, + dirname, + getPrebuilt, + getSources, + join, + parseJSONC, + ROOT_PATH, + walk, +} from "./util.js"; import { checkCopyright } from "./copyright_checker.js"; import * as ciFile from "../.github/workflows/ci.generate.ts"; @@ -25,6 +34,7 @@ if (js) { promises.push(dlintPreferPrimordials()); promises.push(ensureCiYmlUpToDate()); promises.push(ensureNoNewITests()); + promises.push(ensureNoUnusedOutFiles()); if (rs) { promises.push(checkCopyright()); @@ -51,17 +61,20 @@ async function dlint() { ":!:cli/bench/testdata/express-router.js", ":!:cli/bench/testdata/react-dom.js", ":!:cli/compilers/wasm_wrap.js", + ":!:cli/tools/doc/prism.css", + ":!:cli/tools/doc/prism.js", ":!:cli/tsc/dts/**", ":!:cli/tsc/*typescript.js", ":!:cli/tsc/compiler.d.ts", ":!:runtime/examples/", ":!:target/", + ":!:tests/ffi/tests/test.js", ":!:tests/registry/**", ":!:tests/specs/**", ":!:tests/testdata/**", ":!:tests/unit_node/testdata/**", - ":!:tests/wpt/suite/**", ":!:tests/wpt/runner/**", + ":!:tests/wpt/suite/**", ]); if (!sourceFiles.length) { @@ -198,7 +211,7 @@ async function ensureNoNewITests() { "bench_tests.rs": 0, "cache_tests.rs": 0, "cert_tests.rs": 0, - "check_tests.rs": 18, + "check_tests.rs": 2, "compile_tests.rs": 0, "coverage_tests.rs": 0, "eval_tests.rs": 0, @@ -214,13 +227,13 @@ async function ensureNoNewITests() { "lsp_tests.rs": 0, "node_compat_tests.rs": 0, "node_unit_tests.rs": 2, - "npm_tests.rs": 89, + "npm_tests.rs": 5, "pm_tests.rs": 0, "publish_tests.rs": 0, "repl_tests.rs": 0, - "run_tests.rs": 331, + "run_tests.rs": 18, "shared_library_tests.rs": 0, - "task_tests.rs": 3, + "task_tests.rs": 2, "test_tests.rs": 0, "upgrade_tests.rs": 0, "vendor_tests.rs": 1, @@ -248,3 +261,49 @@ async function ensureNoNewITests() { } } } + +async function ensureNoUnusedOutFiles() { + const specsDir = join(ROOT_PATH, "tests", "specs"); + const outFilePaths = new Set( + (await Array.fromAsync( + walk(specsDir, { exts: [".out"] }), + )).map((entry) => entry.path), + ); + const testFiles = (await Array.fromAsync( + walk(specsDir, { exts: [".jsonc"] }), + )).filter((entry) => { + return entry.path.endsWith("__test__.jsonc"); + }); + + function checkObject(baseDirPath, obj) { + for (const [key, value] of Object.entries(obj)) { + if (typeof value === "object") { + checkObject(baseDirPath, value); + } else if (key === "output" && typeof value === "string") { + const outFilePath = join(baseDirPath, value); + outFilePaths.delete(outFilePath); + } + } + } + + for (const testFile of testFiles) { + try { + const text = await Deno.readTextFile(testFile.path); + const data = parseJSONC(text); + checkObject(dirname(testFile.path), data); + } catch (err) { + throw new Error("Failed reading: " + testFile.path, { + cause: err, + }); + } + } + + const notFoundPaths = Array.from(outFilePaths); + if (notFoundPaths.length > 0) { + notFoundPaths.sort(); // be deterministic + for (const file of notFoundPaths) { + console.error(`Unreferenced .out file: ${file}`); + } + throw new Error(`${notFoundPaths.length} unreferenced .out files`); + } +} diff --git a/tools/napi/generate_symbols_lists.js b/tools/napi/generate_symbols_lists.js index 11cf1c434ae92e..efb0edc0436930 100755 --- a/tools/napi/generate_symbols_lists.js +++ b/tools/napi/generate_symbols_lists.js @@ -1,7 +1,7 @@ #!/usr/bin/env -S deno run --allow-read --allow-write // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import exports from "../../cli/napi/sym/symbol_exports.json" with { +import exports from "../../ext/napi/sym/symbol_exports.json" with { type: "json", }; @@ -17,7 +17,7 @@ const symbolExportLists = { for await (const [os, def] of Object.entries(symbolExportLists)) { const defUrl = new URL( - `../../cli/napi/generated_symbol_exports_list_${os}.def`, + `../../ext/napi/generated_symbol_exports_list_${os}.def`, import.meta.url, ); await Deno.writeTextFile(defUrl.pathname, def, { create: true }); diff --git a/tools/release/promote_to_release.ts b/tools/release/promote_to_release.ts old mode 100644 new mode 100755 index c14b590cabf414..046f4d33a8070f --- a/tools/release/promote_to_release.ts +++ b/tools/release/promote_to_release.ts @@ -5,7 +5,7 @@ import { $ } from "jsr:@david/dax@0.41.0"; import { gray } from "jsr:@std/fmt@1/colors"; -import { patchver } from "jsr:@deno/patchver@0.1.0"; +import { patchver } from "jsr:@deno/patchver@0.2.0"; const SUPPORTED_TARGETS = [ "aarch64-apple-darwin", diff --git a/tools/release/release_doc_template.md b/tools/release/release_doc_template.md index fec1746bfb4b23..b37218533783df 100644 --- a/tools/release/release_doc_template.md +++ b/tools/release/release_doc_template.md @@ -30,7 +30,7 @@ Release checklist: ## Patch release preparation -**If you are cutting a patch release**: First you need to sync commits to the +⛔ **If you are cutting a patch release**: First you need to sync commits to the `v$MINOR_VERSION` branch in the `deno` repo. To do that, you need to cherry-pick commits from the main branch to the @@ -75,13 +75,13 @@ verify on GitHub that everything looks correct. 1. Click on the "Run workflow" button. 1. In the drop down, select the minor branch (`v$MINOR_VERSION`) if doing a patch release or the main branch if doing a minor release. - 1. For the kind of release, select either "patch", "minor", or "major". + 1. For the kind of release, select either `patch` or `minor`. 1. Run the workflow. - [ ] Wait for the workflow to complete and for a pull request to be automatically opened. Review the pull request, make any necessary changes, and merge it. - - ⛔ DO NOT create a release tag manually That will automatically happen. + - ⛔ **DO NOT** create a release tag manually That will automatically happen.

Failure Steps @@ -120,7 +120,8 @@ verify on GitHub that everything looks correct. (https://github.com/denoland/deno/releases). - ⛔ Verify that: - - [ ] There are 14 assets on the release draft. + - [ ] There are 24 assets on the + [GitHub release draft](https://github.com/denoland/deno/releases/v$VERSION). - [ ] There are 10 zip files for this version on [dl.deno.land](https://console.cloud.google.com/storage/browser/dl.deno.land/release/v$VERSION). diff --git a/tools/util.js b/tools/util.js index 99133628cee338..8669337bff7ea7 100644 --- a/tools/util.js +++ b/tools/util.js @@ -2,16 +2,24 @@ // deno-lint-ignore-file no-console -import { dirname, fromFileUrl, join, resolve, toFileUrl } from "@std/path"; +import { + dirname, + extname, + fromFileUrl, + join, + resolve, + toFileUrl, +} from "@std/path"; import { wait } from "https://deno.land/x/wait@0.1.13/mod.ts"; -export { dirname, fromFileUrl, join, resolve, toFileUrl }; -export { existsSync, walk } from "@std/fs"; +export { dirname, extname, fromFileUrl, join, resolve, toFileUrl }; +export { existsSync, expandGlobSync, walk } from "@std/fs"; export { TextLineStream } from "@std/streams/text-line-stream"; export { delay } from "@std/async/delay"; +export { parse as parseJSONC } from "@std/jsonc/parse"; // [toolName] --version output const versions = { - "dlint": "dlint 0.60.0", + "dlint": "dlint 0.68.0", }; const compressed = new Set(["ld64.lld", "rcodesign"]); @@ -178,7 +186,7 @@ export function getPrebuiltToolPath(toolName) { return join(PREBUILT_TOOL_DIR, toolName + executableSuffix); } -const commitId = "b8aac22e0cd7c1c6557a56a813fe0c25486fafee"; +const commitId = "7a3a6fee951b3381c59aa4c907274957f324ce8c"; const downloadUrl = `https://raw.githubusercontent.com/denoland/deno_third_party/${commitId}/prebuilt/${platformDirName}`;