Skip to content

Commit

Permalink
Upgrade datafusion to 32.0.0 (#899)
Browse files Browse the repository at this point in the history 
* Upgrade datafusion to 32.0.0

* chore: make cargo fmt happy
  • Loading branch information
@r4ntix
r4ntix authored Nov 16, 2023
1 parent 1063e0a commit c0561ed
Show file tree
Hide file tree
Showing 20 changed files with 95 additions and 75 deletions.
18 changes: 9 additions & 9 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,18 +29,18 @@ members = [
resolver = "2"

[workspace.dependencies]
arrow = { version = "46.0.0" }
arrow-flight = { version = "46.0.0", features = ["flight-sql-experimental"] }
arrow-schema = { version = "46.0.0", default-features = false }
arrow = { version = "47.0.0" }
arrow-flight = { version = "47.0.0", features = ["flight-sql-experimental"] }
arrow-schema = { version = "47.0.0", default-features = false }
configure_me = { version = "0.4.0" }
configure_me_codegen = { version = "0.4.4" }
datafusion = "31.0.0"
datafusion-cli = "31.0.0"
datafusion-proto = "31.0.0"
datafusion = "32.0.0"
datafusion-cli = "32.0.0"
datafusion-proto = "32.0.0"
object_store = "0.7.0"
sqlparser = "0.37.0"
tonic = { version = "0.9" }
tonic-build = { version = "0.9", default-features = false, features = [
sqlparser = "0.38.0"
tonic = { version = "0.10" }
tonic-build = { version = "0.10", default-features = false, features = [
"transport",
"prost"
] }
Expand Down
2 changes: 2 additions & 0 deletions ballista-cli/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ use ballista_cli::{
exec, print_format::PrintFormat, print_options::PrintOptions, BALLISTA_CLI_VERSION,
};
use clap::Parser;
use datafusion_cli::print_options::MaxRows;
use mimalloc::MiMalloc;

#[global_allocator]
Expand Down Expand Up @@ -133,6 +134,7 @@ pub async fn main() -> Result<()> {
let mut print_options = PrintOptions {
format: args.format,
quiet: args.quiet,
maxrows: MaxRows::Unlimited,
};

let files = args.file;
Expand Down
2 changes: 1 addition & 1 deletion ballista/client/src/context.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,7 @@ impl BallistaContext {
// the show tables、 show columns sql can not run at scheduler because the tables is store at client
if is_show {
let state = self.state.lock();
ctx = Arc::new(SessionContext::with_config(
ctx = Arc::new(SessionContext::new_with_config(
SessionConfig::new().with_information_schema(
state.config.default_with_information_schema(),
),
Expand Down
4 changes: 2 additions & 2 deletions ballista/core/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,8 @@ once_cell = "1.9.0"

parking_lot = "0.12"
parse_arg = "0.1.3"
prost = "0.11"
prost-types = "0.11"
prost = "0.12"
prost-types = "0.12"
rand = "0.8"
serde = { version = "1", features = ["derive"] }
sqlparser = { workspace = true }
Expand Down
51 changes: 33 additions & 18 deletions ballista/core/src/serde/generated/ballista.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2025,7 +2025,9 @@ pub mod scheduler_grpc_server {
request: tonic::Request<super::PollWorkParams>,
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move { (*inner).poll_work(request).await };
let fut = async move {
<T as SchedulerGrpc>::poll_work(&inner, request).await
};
Box::pin(fut)
}
}
Expand Down Expand Up @@ -2070,7 +2072,8 @@ pub mod scheduler_grpc_server {
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
(*inner).register_executor(request).await
<T as SchedulerGrpc>::register_executor(&inner, request)
.await
};
Box::pin(fut)
}
Expand Down Expand Up @@ -2116,7 +2119,11 @@ pub mod scheduler_grpc_server {
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
(*inner).heart_beat_from_executor(request).await
<T as SchedulerGrpc>::heart_beat_from_executor(
&inner,
request,
)
.await
};
Box::pin(fut)
}
Expand Down Expand Up @@ -2162,7 +2169,8 @@ pub mod scheduler_grpc_server {
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
(*inner).update_task_status(request).await
<T as SchedulerGrpc>::update_task_status(&inner, request)
.await
};
Box::pin(fut)
}
Expand Down Expand Up @@ -2208,7 +2216,8 @@ pub mod scheduler_grpc_server {
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
(*inner).get_file_metadata(request).await
<T as SchedulerGrpc>::get_file_metadata(&inner, request)
.await
};
Box::pin(fut)
}
Expand Down Expand Up @@ -2254,7 +2263,7 @@ pub mod scheduler_grpc_server {
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
(*inner).create_session(request).await
<T as SchedulerGrpc>::create_session(&inner, request).await
};
Box::pin(fut)
}
Expand Down Expand Up @@ -2300,7 +2309,7 @@ pub mod scheduler_grpc_server {
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
(*inner).update_session(request).await
<T as SchedulerGrpc>::update_session(&inner, request).await
};
Box::pin(fut)
}
Expand Down Expand Up @@ -2346,7 +2355,7 @@ pub mod scheduler_grpc_server {
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
(*inner).remove_session(request).await
<T as SchedulerGrpc>::remove_session(&inner, request).await
};
Box::pin(fut)
}
Expand Down Expand Up @@ -2392,7 +2401,7 @@ pub mod scheduler_grpc_server {
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
(*inner).execute_query(request).await
<T as SchedulerGrpc>::execute_query(&inner, request).await
};
Box::pin(fut)
}
Expand Down Expand Up @@ -2438,7 +2447,7 @@ pub mod scheduler_grpc_server {
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
(*inner).get_job_status(request).await
<T as SchedulerGrpc>::get_job_status(&inner, request).await
};
Box::pin(fut)
}
Expand Down Expand Up @@ -2484,7 +2493,8 @@ pub mod scheduler_grpc_server {
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
(*inner).executor_stopped(request).await
<T as SchedulerGrpc>::executor_stopped(&inner, request)
.await
};
Box::pin(fut)
}
Expand Down Expand Up @@ -2529,7 +2539,9 @@ pub mod scheduler_grpc_server {
request: tonic::Request<super::CancelJobParams>,
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move { (*inner).cancel_job(request).await };
let fut = async move {
<T as SchedulerGrpc>::cancel_job(&inner, request).await
};
Box::pin(fut)
}
}
Expand Down Expand Up @@ -2574,7 +2586,7 @@ pub mod scheduler_grpc_server {
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
(*inner).clean_job_data(request).await
<T as SchedulerGrpc>::clean_job_data(&inner, request).await
};
Box::pin(fut)
}
Expand Down Expand Up @@ -2782,7 +2794,9 @@ pub mod executor_grpc_server {
request: tonic::Request<super::LaunchTaskParams>,
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move { (*inner).launch_task(request).await };
let fut = async move {
<T as ExecutorGrpc>::launch_task(&inner, request).await
};
Box::pin(fut)
}
}
Expand Down Expand Up @@ -2827,7 +2841,8 @@ pub mod executor_grpc_server {
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
(*inner).launch_multi_task(request).await
<T as ExecutorGrpc>::launch_multi_task(&inner, request)
.await
};
Box::pin(fut)
}
Expand Down Expand Up @@ -2873,7 +2888,7 @@ pub mod executor_grpc_server {
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
(*inner).stop_executor(request).await
<T as ExecutorGrpc>::stop_executor(&inner, request).await
};
Box::pin(fut)
}
Expand Down Expand Up @@ -2919,7 +2934,7 @@ pub mod executor_grpc_server {
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
(*inner).cancel_tasks(request).await
<T as ExecutorGrpc>::cancel_tasks(&inner, request).await
};
Box::pin(fut)
}
Expand Down Expand Up @@ -2965,7 +2980,7 @@ pub mod executor_grpc_server {
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
(*inner).remove_job_data(request).await
<T as ExecutorGrpc>::remove_job_data(&inner, request).await
};
Box::pin(fut)
}
Expand Down
10 changes: 8 additions & 2 deletions ballista/core/src/serde/scheduler/to_proto.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,12 +158,18 @@ impl TryInto<protobuf::OperatorMetric> for &MetricValue {
}),
MetricValue::StartTimestamp(timestamp) => Ok(protobuf::OperatorMetric {
metric: Some(operator_metric::Metric::StartTimestamp(
timestamp.value().map(|m| m.timestamp_nanos()).unwrap_or(0),
timestamp
.value()
.and_then(|m| m.timestamp_nanos_opt())
.unwrap_or(0),
)),
}),
MetricValue::EndTimestamp(timestamp) => Ok(protobuf::OperatorMetric {
metric: Some(operator_metric::Metric::EndTimestamp(
timestamp.value().map(|m| m.timestamp_nanos()).unwrap_or(0),
timestamp
.value()
.and_then(|m| m.timestamp_nanos_opt())
.unwrap_or(0),
)),
}),
}
Expand Down
6 changes: 3 additions & 3 deletions ballista/core/src/utils.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ use tonic::transport::{Channel, Error, Server};

/// Default session builder using the provided configuration
pub fn default_session_builder(config: SessionConfig) -> SessionState {
SessionState::with_config_rt(
SessionState::new_with_config_rt(
config,
Arc::new(
RuntimeEnv::new(with_object_store_registry(RuntimeConfig::default()))
Expand Down Expand Up @@ -244,7 +244,7 @@ pub fn create_df_ctx_with_ballista_query_planner<T: 'static + AsLogicalPlan>(
let session_config = SessionConfig::new()
.with_target_partitions(config.default_shuffle_partitions())
.with_information_schema(true);
let mut session_state = SessionState::with_config_rt(
let mut session_state = SessionState::new_with_config_rt(
session_config,
Arc::new(
RuntimeEnv::new(with_object_store_registry(RuntimeConfig::default()))
Expand All @@ -254,7 +254,7 @@ pub fn create_df_ctx_with_ballista_query_planner<T: 'static + AsLogicalPlan>(
.with_query_planner(planner);
session_state = session_state.with_session_id(session_id);
// the SessionContext created here is the client side context, but the session_id is from server side.
SessionContext::with_state(session_state)
SessionContext::new_with_state(session_state)
}

pub struct BallistaQueryPlanner<T: AsLogicalPlan> {
Expand Down
1 change: 1 addition & 0 deletions ballista/executor/src/executor_process.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,7 @@ pub async fn start_executor_process(opt: Arc<ExecutorProcessConfig>) -> Result<(
Some(Arc::new(RuntimeEnv {
memory_pool: runtime.memory_pool.clone(),
disk_manager: runtime.disk_manager.clone(),
cache_manager: runtime.cache_manager.clone(),
object_store_registry: registry,
}))
} else {
Expand Down
4 changes: 2 additions & 2 deletions ballista/executor/src/executor_server.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -509,7 +509,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskRunnerPool<T,
if let Some(task_status) = maybe_task_status {
let task_status_vec = curator_task_status_map
.entry(task_status.scheduler_id)
.or_insert_with(Vec::new);
.or_default();
task_status_vec.push(task_status.task_status);
fetched_task_num += 1;
} else {
Expand All @@ -524,7 +524,7 @@ impl<T: 'static + AsLogicalPlan, U: 'static + AsExecutionPlan> TaskRunnerPool<T,
Ok(task_status) => {
let task_status_vec = curator_task_status_map
.entry(task_status.scheduler_id)
.or_insert_with(Vec::new);
.or_default();
task_status_vec.push(task_status.task_status);
fetched_task_num += 1;
}
Expand Down
6 changes: 3 additions & 3 deletions ballista/scheduler/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ arrow-flight = { workspace = true }
async-recursion = "1.0.0"
async-trait = "0.1.41"
ballista-core = { path = "../core", version = "0.11.0", features = ["s3"] }
base64 = { version = "0.13", default-features = false }
base64 = { version = "0.13" }
clap = { version = "3", features = ["derive", "cargo"] }
configure_me = { workspace = true }
dashmap = "5.4.0"
Expand All @@ -67,8 +67,8 @@ once_cell = { version = "1.16.0", optional = true }
parking_lot = "0.12"
parse_arg = "0.1.3"
prometheus = { version = "0.13", features = ["process"], optional = true }
prost = "0.11"
prost-types = { version = "0.11.0" }
prost = "0.12"
prost-types = { version = "0.12.0" }
rand = "0.8"
serde = { version = "1", features = ["derive"] }
sled_package = { package = "sled", version = "0.34", optional = true }
Expand Down
12 changes: 6 additions & 6 deletions ballista/scheduler/src/flight_sql.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

use arrow_flight::flight_descriptor::DescriptorType;
use arrow_flight::flight_service_server::FlightService;
use arrow_flight::sql::server::FlightSqlService;
use arrow_flight::sql::server::{FlightSqlService, PeekableFlightDataStream};
use arrow_flight::sql::{
ActionBeginSavepointRequest, ActionBeginSavepointResult,
ActionBeginTransactionRequest, ActionBeginTransactionResult,
Expand All @@ -35,6 +35,7 @@ use arrow_flight::{
Action, FlightData, FlightDescriptor, FlightEndpoint, FlightInfo, HandshakeRequest,
HandshakeResponse, Location, Ticket,
};
use futures::Stream;
use log::{debug, error, warn};
use std::convert::TryFrom;
use std::pin::Pin;
Expand Down Expand Up @@ -71,7 +72,6 @@ use prost::Message;
use tokio::sync::mpsc::{channel, Receiver, Sender};
use tokio::time::sleep;
use tokio_stream::wrappers::ReceiverStream;
use tonic::codegen::futures_core::Stream;
use tonic::metadata::MetadataValue;
use uuid::Uuid;

Expand Down Expand Up @@ -846,15 +846,15 @@ impl FlightSqlService for FlightSqlServiceImpl {
async fn do_put_statement_update(
&self,
_ticket: CommandStatementUpdate,
_request: Request<Streaming<FlightData>>,
_request: Request<PeekableFlightDataStream>,
) -> Result<i64, Status> {
debug!("do_put_statement_update");
Err(Status::unimplemented("Implement do_put_statement_update"))
}
async fn do_put_prepared_statement_query(
&self,
_query: CommandPreparedStatementQuery,
_request: Request<Streaming<FlightData>>,
_request: Request<PeekableFlightDataStream>,
) -> Result<Response<<Self as FlightService>::DoPutStream>, Status> {
debug!("do_put_prepared_statement_query");
Err(Status::unimplemented(
Expand All @@ -864,7 +864,7 @@ impl FlightSqlService for FlightSqlServiceImpl {
async fn do_put_prepared_statement_update(
&self,
handle: CommandPreparedStatementUpdate,
request: Request<Streaming<FlightData>>,
request: Request<PeekableFlightDataStream>,
) -> Result<i64, Status> {
debug!("do_put_prepared_statement_update");
let ctx = self.get_ctx(&request)?;
Expand Down Expand Up @@ -927,7 +927,7 @@ impl FlightSqlService for FlightSqlServiceImpl {
async fn do_put_substrait_plan(
&self,
_query: CommandStatementSubstraitPlan,
_request: Request<Streaming<FlightData>>,
_request: Request<PeekableFlightDataStream>,
) -> Result<i64, Status> {
debug!("do_put_substrait_plan");
Err(Status::unimplemented("Implement do_put_substrait_plan"))
Expand Down
Loading

0 comments on commit c0561ed

Please sign in to comment.