Minor: Move depcheck out of datafusion crate (200 less crates to comp…

…ile) (apache#9865)

* Minor: Move depcheck out of main datafusion test

* Update dev/depcheck/README.md

Co-authored-by: Andrew Lamb <[email protected]>

---------

Co-authored-by: comphead <[email protected]>

.github/workflows/rust.yml

@@ -195,6 +195,25 @@ jobs:
       - name: Verify Working Directory Clean
         run: git diff --exit-code
 
+  depcheck:
+    name: circular dependency check
+    needs: [ linux-build-lib ]
+    runs-on: ubuntu-latest
+    container:
+      image: amd64/rust
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: stable
+      - name: Check dependencies
+        run: |
+          cd dev/depcheck
+          cargo run 
+
   # Run `cargo test doc` (test documentation examples)
   linux-test-doc:
     name: cargo test doc (amd64)

Cargo.toml

@@ -16,7 +16,7 @@
 # under the License.
 
 [workspace]
-exclude = ["datafusion-cli"]
+exclude = ["datafusion-cli", "dev/depcheck"]
 members = [
     "datafusion/common",
     "datafusion/common-runtime",

datafusion/core/Cargo.toml

@@ -130,7 +130,6 @@ zstd = { version = "0.13", optional = true, default-features = false }
 [dev-dependencies]
 async-trait = { workspace = true }
 bigdecimal = { workspace = true }
-cargo = "0.78.1"
 criterion = { version = "0.5", features = ["async_tokio"] }
 csv = "1.1.6"
 ctor = { workspace = true }

dev/depcheck/Cargo.toml

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Circular dependency checker for DataFusion
+[package]
+name = "depcheck"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+cargo = "0.78.1"

dev/depcheck/README.md

@@ -0,0 +1,26 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+This directory contains a tool that ensures there are no circular dependencies
+in the DataFusion codebase.
+
+Specifically, it checks that no create's tests depend on another crate which
+depends on the first, which prevents publishing to crates.io, for example
+
+[issue 9272]: https://github.com/apache/arrow-datafusion/issues/9277:

datafusion/core/tests/depcheck.rs → dev/depcheck/src/main.rs

@@ -15,18 +15,38 @@
 // specific language governing permissions and limitations
 // under the License.
 
+extern crate cargo;
+
+use cargo::CargoResult;
 /// Check for circular dependencies between DataFusion crates
 use std::collections::{HashMap, HashSet};
 use std::env;
 use std::path::Path;
 
 use cargo::util::config::Config;
-#[test]
-fn test_deps() -> Result<(), Box<dyn std::error::Error>> {
+
+/// Verifies that there are no circular dependencies between DataFusion crates
+/// (which prevents publishing on crates.io) by parsing the Cargo.toml files and
+/// checking the dependency graph.
+///
+/// See https://github.com/apache/arrow-datafusion/issues/9278 for more details
+fn main() -> CargoResult<()> {
     let config = Config::default()?;
+    // This is the path for the depcheck binary
     let path = env::var("CARGO_MANIFEST_DIR").unwrap();
-    let dir = Path::new(&path);
-    let root_cargo_toml = dir.join("Cargo.toml");
+    let root_cargo_toml = Path::new(&path)
+        // dev directory
+        .parent()
+        .expect("Can not find dev directory")
+        // project root directory
+        .parent()
+        .expect("Can not find project root directory")
+        .join("Cargo.toml");
+
+    println!(
+        "Checking for circular dependencies in {}",
+        root_cargo_toml.display()
+    );
     let workspace = cargo::core::Workspace::new(&root_cargo_toml, &config)?;
     let (_, resolve) = cargo::ops::resolve_ws(&workspace)?;
 
@@ -50,7 +70,7 @@ fn test_deps() -> Result<(), Box<dyn std::error::Error>> {
             check_circular_deps(root_package, dep, &package_deps, &mut seen);
         }
     }
-
+    println!("No circular dependencies found");
     Ok(())
 }